From 2eab8b7aa95e94e631813b2e85d419c91a075613 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 25 Jun 2024 16:16:11 -0400 Subject: [PATCH] make longterm monitoring deployable on a different server --- birdhouse/optional-components/README.rst | 18 +++++++++++++----- .../config/monitoring/docker-compose-extra.yml | 6 ------ .../config/monitoring/prometheus.null.rules | 4 ---- .../prometheus-longterm-metrics/default.env | 17 ++++++++++++----- .../docker-compose-extra.yml | 2 +- .../prometheus.yml.template | 3 +-- .../config/monitoring/docker-compose-extra.yml | 6 ++++++ .../config/monitoring/prometheus.rules | 0 8 files changed, 33 insertions(+), 23 deletions(-) delete mode 100644 birdhouse/optional-components/prometheus-longterm-metrics/config/monitoring/docker-compose-extra.yml delete mode 100644 birdhouse/optional-components/prometheus-longterm-metrics/config/monitoring/prometheus.null.rules create mode 100644 birdhouse/optional-components/prometheus-longterm-rules/config/monitoring/docker-compose-extra.yml rename birdhouse/optional-components/{prometheus-longterm-metrics => prometheus-longterm-rules}/config/monitoring/prometheus.rules (100%) diff --git a/birdhouse/optional-components/README.rst b/birdhouse/optional-components/README.rst index 7a4bf4ec..583cbe16 100644 --- a/birdhouse/optional-components/README.rst +++ b/birdhouse/optional-components/README.rst @@ -451,12 +451,11 @@ This is a second prometheus instance that collects longterm monitoring metrics f (the one created by the ``components/monitoring`` component). Longterm metrics are any prometheus rule that have the label ``group: longterm-metrics`` or in other words are -selectable using prometheus' ``'{group="longterm-metrics"}'`` query filter. To see which longterm metric rules are -added by default see the ``optional-components/prometheus-longterm-metrics/config/monitoring/prometheus.rules.template``. +selectable using prometheus' ``'{group="longterm-metrics"}'`` query filter. To add some default longterm metrics rules +also enable the ``prometheus-longterm-rules`` component. -If you do not want the default longterm-metric rules included, set the ``PROMETHEUS_LONGTERM_RULES_FILE`` to anything -other than ``True`` in your ``env.local`` file. You may want to do this if you've created your own set of rules in -another component that you would like to use instead of the default ones. +You may also choose to create your own set of rules in another component that you would like to use instead of the +default ones. To configure this component: @@ -464,6 +463,15 @@ To configure this component: Enabling this component creates the additional endpoint ``/prometheus-longterm-metrics``. +Prometheus Long-term Rules +-------------------------- + +This adds some default longterm metrics rules to the `prometheus` component for use by the `prometheus-longterm-metrics` +component. These rules all have the label ``group: longterm-metrics``. + +To see which rules are added, check out the +`optional-components/prometheus-longterm-rules/config/monitoring/prometheus.rules` file. + Thanos ------ diff --git a/birdhouse/optional-components/prometheus-longterm-metrics/config/monitoring/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-longterm-metrics/config/monitoring/docker-compose-extra.yml deleted file mode 100644 index 3eaa00e2..00000000 --- a/birdhouse/optional-components/prometheus-longterm-metrics/config/monitoring/docker-compose-extra.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3.4" - -services: - prometheus: - volumes: - - ./optional-components/prometheus-longterm-metrics/config/monitoring/${PROMETHEUS_LONGTERM_RULES_FILE}:/etc/prometheus/prometheus-longterm-metrics.rules:ro diff --git a/birdhouse/optional-components/prometheus-longterm-metrics/config/monitoring/prometheus.null.rules b/birdhouse/optional-components/prometheus-longterm-metrics/config/monitoring/prometheus.null.rules deleted file mode 100644 index a885bf2e..00000000 --- a/birdhouse/optional-components/prometheus-longterm-metrics/config/monitoring/prometheus.null.rules +++ /dev/null @@ -1,4 +0,0 @@ -# This file is intentionally left blank in order to allow a user to choose whether to enable the default rules that are -# set in the prometheus.rules file. -# By setting the PROMETHEUS_LONGTERM_ENABLE_DEFAULT_RULES environment variable to True, the rules in prometheus.rules -# will be added. By setting that value to anything else, this file will be added instead. diff --git a/birdhouse/optional-components/prometheus-longterm-metrics/default.env b/birdhouse/optional-components/prometheus-longterm-metrics/default.env index 75c6c46d..3e3b5aa7 100644 --- a/birdhouse/optional-components/prometheus-longterm-metrics/default.env +++ b/birdhouse/optional-components/prometheus-longterm-metrics/default.env @@ -1,23 +1,30 @@ +export PROMETHEUS_LONGTERM_VERSION='${PROMETHEUS_VERSION:-"v2.52.0"}' +export PROMETHEUS_LONGTERM_DOCKER='${PROMETHEUS_DOCKER:-prom/prometheus}' +export PROMETHEUS_LONGTERM_IMAGE='${PROMETHEUS_LONGTERM_DOCKER}:${PROMETHEUS_LONGTERM_VERSION}' + export PROMETHEUS_LONGTERM_RETENTION_TIME=1y -export PROMETHEUS_LONGTERM_ENABLE_DEFAULT_RULES=True export PROMETHEUS_LONGTERM_SCRAPE_INTERVAL=1h # These are the prometheus defaults export PROMETHEUS_LONGTERM_TSDB_MIN_BLOCK_DURATION=2h export PROMETHEUS_LONGTERM_TSDB_MAX_BLOCK_DURATION=1d12h -export PROMETHEUS_LONGTERM_RULES_FILE='$([ "${PROMETHEUS_LONGTERM_ENABLE_DEFAULT_RULES}" = "True" ] && echo prometheus.rules || echo prometheus.null.rules)' +# These are the targets that +export PROMETHEUS_LONGTERM_TARGETS='["prometheus:9090"]' # yaml list syntax OPTIONAL_VARS=" $OPTIONAL_VARS \$PROMETHEUS_LONGTERM_SCRAPE_INTERVAL + \$PROMETHEUS_LONGTERM_TARGETS " export DELAYED_EVAL=" $DELAYED_EVAL + PROMETHEUS_LONGTERM_VERSION + PROMETHEUS_LONGTERM_DOCKER + PROMETHEUS_LONGTERM_IMAGE PROMETHEUS_LONGTERM_RULES_FILE " -COMPONENT_DEPENDENCIES=" - ./components/monitoring -" +# Note that this component does not depend explicitly on the `components/monitoring` component so that this can +# theoretically be deployed on a different machine than the `prometheus` service. This is currently untested. diff --git a/birdhouse/optional-components/prometheus-longterm-metrics/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-longterm-metrics/docker-compose-extra.yml index 49c6f152..426d4d0e 100644 --- a/birdhouse/optional-components/prometheus-longterm-metrics/docker-compose-extra.yml +++ b/birdhouse/optional-components/prometheus-longterm-metrics/docker-compose-extra.yml @@ -9,7 +9,7 @@ x-logging: services: prometheus-longterm-metrics: - image: ${PROMETHEUS_IMAGE} + image: ${PROMETHEUS_LONGTERM_IMAGE} container_name: prometheus-longterm-metrics volumes: - ./optional-components/prometheus-longterm-metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro diff --git a/birdhouse/optional-components/prometheus-longterm-metrics/prometheus.yml.template b/birdhouse/optional-components/prometheus-longterm-metrics/prometheus.yml.template index 5ca24f00..c0ade1ba 100644 --- a/birdhouse/optional-components/prometheus-longterm-metrics/prometheus.yml.template +++ b/birdhouse/optional-components/prometheus-longterm-metrics/prometheus.yml.template @@ -14,5 +14,4 @@ scrape_configs: - '{group="longterm-metrics"}' static_configs: - - targets: - - 'prometheus:9090' + - targets: ${PROMETHEUS_LONGTERM_TARGETS} diff --git a/birdhouse/optional-components/prometheus-longterm-rules/config/monitoring/docker-compose-extra.yml b/birdhouse/optional-components/prometheus-longterm-rules/config/monitoring/docker-compose-extra.yml new file mode 100644 index 00000000..0f701b30 --- /dev/null +++ b/birdhouse/optional-components/prometheus-longterm-rules/config/monitoring/docker-compose-extra.yml @@ -0,0 +1,6 @@ +version: "3.4" + +services: + prometheus: + volumes: + - ./optional-components/prometheus-longterm-rules/config/monitoring/prometheus.rules:/etc/prometheus/prometheus-longterm-metrics.rules:ro diff --git a/birdhouse/optional-components/prometheus-longterm-metrics/config/monitoring/prometheus.rules b/birdhouse/optional-components/prometheus-longterm-rules/config/monitoring/prometheus.rules similarity index 100% rename from birdhouse/optional-components/prometheus-longterm-metrics/config/monitoring/prometheus.rules rename to birdhouse/optional-components/prometheus-longterm-rules/config/monitoring/prometheus.rules