diff --git a/.env.example b/.env.example index df0eac4f8db..bfb09961180 100644 --- a/.env.example +++ b/.env.example @@ -22,7 +22,9 @@ PROMETHEUS_ALERTS_DIR=monitoring/ceph-mixin #PROMETHEUS_ALERTS_DIR=monitoring/prometheus/alerts PROMETHEUS_HOST_PORT=9090 +PROMETHEUS_HOST_PORT1=9098 #PROMETHEUS_IMAGE= +THANOS_IMAGE=thanosio/thanos:main-2023-10-16-77fac93 NODE_EXPORTER_HOST_PORT=9100 #NODE_EXPORTER_IMAGE= ALERTMANAGER_HOST_PORT=9093 diff --git a/docker-compose.yml b/docker-compose.yml index 9727a603737..e77a2659c5b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -121,6 +121,63 @@ services: - ./docker/prometheus:/etc/prometheus:Z - ${CEPH_REPO_DIR}/${PROMETHEUS_ALERTS_DIR:-monitoring/ceph-mixin}:/etc/prometheus/alerts:Z + prometheus2: + image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.43.0} + container_name: prometheus2 + hostname: prometheus2 + ports: ['${PROMETHEUS2_HOST_PORT:-9090}:9090'] + volumes: + - ./docker/prometheus1:/etc/prometheus:Z + - ${CEPH2_REPO_DIR}/${PROMETHEUS_ALERTS_DIR:-monitoring/ceph-mixin}:/etc/prometheus/alerts:Z + + thanos-sidecar: + image: ${THANOS_IMAGE} + container_name: thanos-sidecar + hostname: thanos-sidecar + volumes_from: + - prometheus:rw + command: + - "sidecar" + - "--tsdb.path=/prometheus" + - "--prometheus.url=http://prometheus:9090" + expose: + - 10901 + - 10902 + depends_on: + - prometheus + + thanos-sidecar2: + image: ${THANOS_IMAGE} + container_name: thanos-sidecar2 + hostname: thanos-sidecar2 + volumes_from: + - prometheus2:rw + command: + - "sidecar" + - "--tsdb.path=/prometheus" + - "--prometheus.url=http://prometheus2:9090" + expose: + - 10901 + - 10902 + depends_on: + - prometheus2 + + thanos-querier: + image: ${THANOS_IMAGE} + container_name: thanos-querier + hostname: thanos-querier + command: + - "query" + - "--store=thanos-sidecar:10901" + - "--store=thanos-sidecar2:10901" + expose: + - 10902 + - 10901 + ports: + - "10902:10902" + depends_on: + - thanos-sidecar + node-exporter: image: ${NODE_EXPORTER_IMAGE:-prom/node-exporter:v1.5.0} container_name: node-exporter diff --git a/docker/grafana/provisioning/datasources/prometheus.yml b/docker/grafana/provisioning/datasources/prometheus.yml index 1b52e798ab3..6910e092702 100644 --- a/docker/grafana/provisioning/datasources/prometheus.yml +++ b/docker/grafana/provisioning/datasources/prometheus.yml @@ -11,3 +11,10 @@ datasources: orgId: 1 url: http://prometheus:9090 editable: true + + - name: Thanos + type: prometheus + access: proxy + orgId: 1 + url: http://thanos-querier:10902 + editable: true diff --git a/docker/prometheus/ceph-targets.yml b/docker/prometheus/ceph-targets.yml index c0114b2912f..cf4469d0bd3 100644 --- a/docker/prometheus/ceph-targets.yml +++ b/docker/prometheus/ceph-targets.yml @@ -1,10 +1,8 @@ [ { "targets": [ "ceph:9283" ], - "labels": {} + "labels": { + cluster: "cluster1" + } }, - { - "targets": [ "ceph2:9283" ], - "labels": {} - } -] \ No newline at end of file +] diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 670feca365c..75d0b62f9fb 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -1,7 +1,8 @@ global: scrape_interval: 5s evaluation_interval: 5s - + external_labels: + cluster: cluster1 scrape_configs: - job_name: 'node-exporter' file_sd_configs: diff --git a/docker/prometheus1/ceph-targets.yml b/docker/prometheus1/ceph-targets.yml new file mode 100644 index 00000000000..93b4b6a9549 --- /dev/null +++ b/docker/prometheus1/ceph-targets.yml @@ -0,0 +1,8 @@ +[ + { + "targets": [ "ceph2:9283" ], + "labels": { + cluster: "cluster2" + } + } +] diff --git a/docker/prometheus1/node-exporter-targets.yml b/docker/prometheus1/node-exporter-targets.yml new file mode 100644 index 00000000000..ea5c0e1a2ce --- /dev/null +++ b/docker/prometheus1/node-exporter-targets.yml @@ -0,0 +1,8 @@ +[ + { + "targets": [ "node-exporter:9100" ], + "labels": { + "instance": "node-exporter" + } + } +] diff --git a/docker/prometheus1/prometheus.yml b/docker/prometheus1/prometheus.yml new file mode 100644 index 00000000000..6ee369bb5a2 --- /dev/null +++ b/docker/prometheus1/prometheus.yml @@ -0,0 +1,26 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + external_labels: + cluster: cluster2 +scrape_configs: + - job_name: 'node-exporter' + file_sd_configs: + - files: + - node-exporter-targets.yml + - job_name: 'ceph' + honor_labels: true + file_sd_configs: + - files: + - ceph-targets.yml + +alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - 'alertmanager:9094' + +rule_files: + - 'alerts/ceph_default_alerts.yml' + - 'alerts/prometheus_alerts.yml'