diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet index 65884bdbd229..e394e979436c 100644 --- a/.drone/drone.jsonnet +++ b/.drone/drone.jsonnet @@ -284,6 +284,9 @@ local manifest(apps) = pipeline('manifest') { image: 'koalaman/shellcheck-alpine:stable', commands: ['apk add make bash && make lint-scripts'], }, + make('loki', container=false) { depends_on: ['clone'] }, + make('validate-example-configs', container=false) { depends_on: ['loki'] }, + make('check-example-config-doc', container=false) { depends_on: ['clone'] }, ], }, ] + [ diff --git a/.drone/drone.yml b/.drone/drone.yml index 8108b45df82f..bb46d9269376 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -46,6 +46,27 @@ steps: commands: - apk add make bash && make lint-scripts +- name: loki + image: grafana/loki-build-image:0.17.0 + commands: + - make BUILD_IN_CONTAINER=false loki + depends_on: + - clone + +- name: validate-example-configs + image: grafana/loki-build-image:0.17.0 + commands: + - make BUILD_IN_CONTAINER=false validate-example-configs + depends_on: + - loki + +- name: check-example-config-doc + image: grafana/loki-build-image:0.17.0 + commands: + - make BUILD_IN_CONTAINER=false check-example-config-doc + depends_on: + - clone + --- kind: pipeline name: docker-amd64 @@ -973,6 +994,6 @@ get: --- kind: signature -hmac: b70be41d1a7f91c11af945a34bdbdc7a4f7613cf830c13f5438dba0bf33a1ec5 +hmac: f1681daaa15376eb15eae5dd623a3ec19544b0447506aba2d59a92f0fd8fee49 ... diff --git a/Makefile b/Makefile index 69c65985483e..2a65a1867bda 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ .PHONY: bigtable-backup, push-bigtable-backup .PHONY: benchmark-store, drone, check-mod .PHONY: migrate migrate-image lint-markdown ragel +.PHONY: validate-example-configs generate-example-config-doc check-example-config-doc SHELL = /usr/bin/env bash @@ -635,3 +636,30 @@ format: -type f -name '*.go' -exec gofmt -w -s {} \; find . $(DONT_FIND) -name '*.pb.go' -prune -o -name '*.y.go' -prune -o -name '*.rl.go' -prune -o \ -type f -name '*.go' -exec goimports -w -local github.com/grafana/loki {} \; + +################### +# Example Configs # +################### + +# Validate the example configurations that we provide in ./docs/sources/configuration/examples +validate-example-configs: loki + for f in ./docs/sources/configuration/examples/*.yaml; do echo "Validating provided example config: $$f" && ./cmd/loki/loki -config.file=$$f -verify-config || exit 1; done + +# Dynamically generate ./docs/sources/configuration/examples.md using the example configs that we provide. +# This target should be run if any of our example configs change. +generate-example-config-doc: + echo "Removing existing doc at loki/docs/configuration/examples.md and re-generating. . ." + # Title and Heading + echo -e "---\ntitle: Examples\n---\n # Loki Configuration Examples" > ./docs/sources/configuration/examples.md + # Append each configuration and its file name to examples.md + for f in ./docs/sources/configuration/examples/*.yaml; do echo -e "\n## $$(basename $$f)\n\n\`\`\`yaml\n$$(cat $$f)\n\`\`\`\n" >> ./docs/sources/configuration/examples.md; done + +# Fail our CI build if changes are made to example configurations but our doc is not updated +check-example-config-doc: generate-example-config-doc + @if ! (git diff --exit-code ./docs/sources/configuration/examples.md); then \ + echo -e "\nChanges found in generated example configuration doc"; \ + echo "Run 'make generate-example-config-doc' and commit the changes to fix this error."; \ + echo "If you are actively developing these files you can ignore this error"; \ + echo -e "(Don't forget to check in the generated files when finished)\n"; \ + exit 1; \ + fi diff --git a/docs/sources/configuration/examples.md b/docs/sources/configuration/examples.md index 939c1aa2a2e1..0a54ff7e9170 100644 --- a/docs/sources/configuration/examples.md +++ b/docs/sources/configuration/examples.md @@ -1,109 +1,112 @@ --- title: Examples --- -# Grafana Loki Configuration Examples + # Loki Configuration Examples -## Complete Local configuration +## almost-zero-dependency.yaml ```yaml +# This is a configuration to deploy Loki depending only on a storage solution +# for example, an S3-compatible API like MinIO. +# The ring configuration is based on the gossip memberlist and the index is shipped to storage +# via Single Store (boltdb-shipper) + auth_enabled: false server: http_listen_port: 3100 +distributor: + ring: + kvstore: + store: memberlist + ingester: lifecycler: - address: 127.0.0.1 ring: kvstore: - store: inmemory + store: memberlist replication_factor: 1 final_sleep: 0s chunk_idle_period: 5m chunk_retain_period: 30s +memberlist: + abort_if_cluster_join_fails: false + + # Expose this port on all distributor, ingester + # and querier replicas. + bind_port: 7946 + + # You can use a headless k8s service for all distributor, + # ingester and querier components. + join_members: + - loki-gossip-ring.loki.svc.cluster.local:7946 + + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + schema_config: configs: - from: 2020-05-15 - store: boltdb - object_store: filesystem + store: boltdb-shipper + object_store: s3 schema: v11 index: prefix: index_ - period: 168h + period: 24h storage_config: - boltdb: - directory: /tmp/loki/index + boltdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + shared_store: s3 - filesystem: - directory: /tmp/loki/chunks + aws: + s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name + s3forcepathstyle: true limits_config: enforce_metric_name: false reject_old_samples: true reject_old_samples_max_age: 168h +compactor: + working_directory: /data/compactor + shared_store: s3 + compaction_interval: 5m ``` -## Google Cloud Storage -This is partial config that uses GCS and Bigtable for the chunk and index -stores, respectively. +## aws-basic-config-no-creds.yaml ```yaml -schema_config: - configs: - - from: 2020-05-15 - store: bigtable - object_store: gcs - schema: v11 - index: - prefix: loki_index_ - period: 168h - -storage_config: - bigtable: - instance: BIGTABLE_INSTANCE - project: BIGTABLE_PROJECT - gcs: - bucket_name: GCS_BUCKET_NAME -``` - -## Cassandra Index +# If you don't wish to hard-code S3 credentials you can also configure an EC2 +# instance role by changing the `storage_config` section -This is a partial config that uses the local filesystem for chunk storage and -Cassandra for the index storage: - -```yaml schema_config: configs: - from: 2020-05-15 - store: cassandra - object_store: filesystem + store: aws + object_store: s3 schema: v11 index: - prefix: cassandra_table - period: 168h - + prefix: loki_ storage_config: - cassandra: - username: cassandra - password: cassandra - addresses: 127.0.0.1 - auth: true - keyspace: lokiindex - - filesystem: - directory: /tmp/loki/chunks + aws: + s3: s3://region/bucket_name + dynamodb: + dynamodb_url: dynamodb://region + ``` -## AWS -This is a partial config that uses S3 for chunk storage and DynamoDB for the -index storage: +## aws-basic-config.yaml ```yaml +# This partial configuration uses S3 for chunk storage and uses DynamoDB for index storage + schema_config: configs: - from: 2020-05-15 @@ -117,59 +120,40 @@ storage_config: s3: s3://access_key:secret_access_key@region/bucket_name dynamodb: dynamodb_url: dynamodb://access_key:secret_access_key@region + ``` -If you don't wish to hard-code S3 credentials, you can also configure an EC2 -instance role by changing the `storage_config` section: - -```yaml -storage_config: - aws: - s3: s3://region/bucket_name - dynamodb: - dynamodb_url: dynamodb://region -``` -### S3-compatible APIs - -S3-compatible APIs (e.g. Ceph Object Storage with an S3-compatible API) can be -used. If the API supports path-style URL rather than virtual hosted bucket -addressing, configure the URL in `storage_config` with the custom endpoint: +## cassandra-index.yaml ```yaml -storage_config: - aws: - s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name - s3forcepathstyle: true -``` - -### S3 Expanded Config +# This is a partial config that uses the local filesystem for chunk storage and Cassandra for index storage -S3 config now supports expanded config. Either `s3` endpoint URL can be used -or expanded config can be used. +schema_config: + configs: + - from: 2020-05-15 + store: cassandra + object_store: filesystem + schema: v11 + index: + prefix: cassandra_table + period: 168h -```yaml storage_config: - aws: - bucketnames: bucket_name1, bucket_name2 - endpoint: s3.endpoint.com - region: s3_region - access_key_id: s3_access_key_id - secret_access_key: s3_secret_access_key - insecure: false - sse_encryption: false - http_config: - idle_conn_timeout: 90s - response_header_timeout: 0s - insecure_skip_verify: false - s3forcepathstyle: true + cassandra: + username: cassandra + password: cassandra + addresses: 127.0.0.1 + auth: true + keyspace: lokiindex + + filesystem: + directory: /tmp/loki/chunks + ``` -## Almost zero dependencies setup -This is a configuration to deploy Loki depending only on storage solution, e.g. an -S3-compatible API like minio. The ring configuration is based on the gossip memberlist -and the index is shipped to storage via [Single Store (boltdb-shipper)](../../operations/storage/boltdb-shipper/). +## complete-local-config.yaml ```yaml auth_enabled: false @@ -177,75 +161,51 @@ auth_enabled: false server: http_listen_port: 3100 -distributor: - ring: - kvstore: - store: memberlist - ingester: lifecycler: + address: 127.0.0.1 ring: kvstore: - store: memberlist + store: inmemory replication_factor: 1 final_sleep: 0s chunk_idle_period: 5m chunk_retain_period: 30s -memberlist: - abort_if_cluster_join_fails: false - - # Expose this port on all distributor, ingester - # and querier replicas. - bind_port: 7946 - - # You can use a headless k8s service for all distributor, - # ingester and querier components. - join_members: - - loki-gossip-ring.loki.svc.cluster.local:7946 - - max_join_backoff: 1m - max_join_retries: 10 - min_join_backoff: 1s - schema_config: configs: - from: 2020-05-15 - store: boltdb-shipper - object_store: s3 + store: boltdb + object_store: filesystem schema: v11 index: prefix: index_ - period: 24h + period: 168h storage_config: - boltdb_shipper: - active_index_directory: /loki/index - cache_location: /loki/index_cache - shared_store: s3 + boltdb: + directory: /tmp/loki/index - aws: - s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name - s3forcepathstyle: true + filesystem: + directory: /tmp/loki/chunks limits_config: enforce_metric_name: false reject_old_samples: true reject_old_samples_max_age: 168h - -compactor: - working_directory: /data/compactor - shared_store: s3 - compaction_interval: 5m ``` -## schema_config + +## example-schema-config.yaml ```yaml -configs: - # Starting from 2018-04-15 Loki should store indexes on Cassandra - # using weekly periodic tables and chunks on filesystem. - # The index tables will be prefixed with "index_". +# Additional example schema configuration for Cassandra + +schema_config: + configs: + # Starting from 2018-04-15 Loki should store indexes on Cassandra + # using weekly periodic tables and chunks on filesystem. + # The index tables will be prefixed with "index_". - from: "2018-04-15" store: cassandra object_store: filesystem @@ -262,4 +222,87 @@ configs: index: period: 168h prefix: index_ + ``` + + +## google-cloud-storage-config.yaml + +```yaml +# This partial configuration uses GCS for chunk storage and uses BigTable for index storage + +schema_config: + configs: + - from: 2020-05-15 + store: bigtable + object_store: gcs + schema: v11 + index: + prefix: loki_index_ + period: 168h + +storage_config: + bigtable: + instance: BIGTABLE_INSTANCE + project: BIGTABLE_PROJECT + gcs: + bucket_name: GCS_BUCKET_NAME + +``` + + +## s3-compatible-apis.yaml + +```yaml +# S3-compatible APIs such as Ceph Object Storage with an S3-compatible API, can be used. +# If the API supports path-style URLs rather than virtual hosted bucket addressing, +# configure the URL in `storage_config` with the custom endpoint + +schema_config: + configs: + - from: 2020-05-15 + store: aws + object_store: s3 + schema: v11 + index: + prefix: loki_ +storage_config: + aws: + s3: s3://access_key:secret_access_key@region/bucket_name + dynamodb: + dynamodb_url: dynamodb://access_key:secret_access_key@region + +``` + + +## s3-expanded-config.yaml + +```yaml +# S3 configuration supports an expanded configuration. +# Either an `s3` endpoint URL can be used, or an expanded configuration can be used. + +schema_config: + configs: + - from: 2020-05-15 + store: aws + object_store: s3 + schema: v11 + index: + prefix: loki_ +storage_config: + aws: + bucketnames: bucket_name1, bucket_name2 + endpoint: s3.endpoint.com + region: s3_region + access_key_id: s3_access_key_id + secret_access_key: s3_secret_access_key + insecure: false + sse_encryption: false + http_config: + idle_conn_timeout: 90s + response_header_timeout: 0s + insecure_skip_verify: false + s3forcepathstyle: true + +``` + diff --git a/docs/sources/configuration/examples/almost-zero-dependency.yaml b/docs/sources/configuration/examples/almost-zero-dependency.yaml new file mode 100644 index 000000000000..c82a3abf8929 --- /dev/null +++ b/docs/sources/configuration/examples/almost-zero-dependency.yaml @@ -0,0 +1,70 @@ +# This is a configuration to deploy Loki depending only on a storage solution +# for example, an S3-compatible API like MinIO. +# The ring configuration is based on the gossip memberlist and the index is shipped to storage +# via Single Store (boltdb-shipper) + +auth_enabled: false + +server: + http_listen_port: 3100 + +distributor: + ring: + kvstore: + store: memberlist + +ingester: + lifecycler: + ring: + kvstore: + store: memberlist + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 5m + chunk_retain_period: 30s + +memberlist: + abort_if_cluster_join_fails: false + + # Expose this port on all distributor, ingester + # and querier replicas. + bind_port: 7946 + + # You can use a headless k8s service for all distributor, + # ingester and querier components. + join_members: + - loki-gossip-ring.loki.svc.cluster.local:7946 + + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s + +schema_config: + configs: + - from: 2020-05-15 + store: boltdb-shipper + object_store: s3 + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + shared_store: s3 + + aws: + s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name + s3forcepathstyle: true + +limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + +compactor: + working_directory: /data/compactor + shared_store: s3 + compaction_interval: 5m diff --git a/docs/sources/configuration/examples/aws-basic-config-no-creds.yaml b/docs/sources/configuration/examples/aws-basic-config-no-creds.yaml new file mode 100644 index 000000000000..98f352f6d3fb --- /dev/null +++ b/docs/sources/configuration/examples/aws-basic-config-no-creds.yaml @@ -0,0 +1,17 @@ +# If you don't wish to hard-code S3 credentials you can also configure an EC2 +# instance role by changing the `storage_config` section + +schema_config: + configs: + - from: 2020-05-15 + store: aws + object_store: s3 + schema: v11 + index: + prefix: loki_ +storage_config: + aws: + s3: s3://region/bucket_name + dynamodb: + dynamodb_url: dynamodb://region + \ No newline at end of file diff --git a/docs/sources/configuration/examples/aws-basic-config.yaml b/docs/sources/configuration/examples/aws-basic-config.yaml new file mode 100644 index 000000000000..0d69a7182ff7 --- /dev/null +++ b/docs/sources/configuration/examples/aws-basic-config.yaml @@ -0,0 +1,16 @@ +# This partial configuration uses S3 for chunk storage and uses DynamoDB for index storage + +schema_config: + configs: + - from: 2020-05-15 + store: aws + object_store: s3 + schema: v11 + index: + prefix: loki_ +storage_config: + aws: + s3: s3://access_key:secret_access_key@region/bucket_name + dynamodb: + dynamodb_url: dynamodb://access_key:secret_access_key@region + \ No newline at end of file diff --git a/docs/sources/configuration/examples/cassandra-index.yaml b/docs/sources/configuration/examples/cassandra-index.yaml new file mode 100644 index 000000000000..03b37682a9d1 --- /dev/null +++ b/docs/sources/configuration/examples/cassandra-index.yaml @@ -0,0 +1,23 @@ +# This is a partial config that uses the local filesystem for chunk storage and Cassandra for index storage + +schema_config: + configs: + - from: 2020-05-15 + store: cassandra + object_store: filesystem + schema: v11 + index: + prefix: cassandra_table + period: 168h + +storage_config: + cassandra: + username: cassandra + password: cassandra + addresses: 127.0.0.1 + auth: true + keyspace: lokiindex + + filesystem: + directory: /tmp/loki/chunks + \ No newline at end of file diff --git a/docs/sources/configuration/examples/complete-local-config.yaml b/docs/sources/configuration/examples/complete-local-config.yaml new file mode 100644 index 000000000000..16942b969466 --- /dev/null +++ b/docs/sources/configuration/examples/complete-local-config.yaml @@ -0,0 +1,37 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +ingester: + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 5m + chunk_retain_period: 30s + +schema_config: + configs: + - from: 2020-05-15 + store: boltdb + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 168h + +storage_config: + boltdb: + directory: /tmp/loki/index + + filesystem: + directory: /tmp/loki/chunks + +limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h diff --git a/docs/sources/configuration/examples/example-schema-config.yaml b/docs/sources/configuration/examples/example-schema-config.yaml new file mode 100644 index 000000000000..618f8dafe442 --- /dev/null +++ b/docs/sources/configuration/examples/example-schema-config.yaml @@ -0,0 +1,24 @@ +# Additional example schema configuration for Cassandra + +schema_config: + configs: + # Starting from 2018-04-15 Loki should store indexes on Cassandra + # using weekly periodic tables and chunks on filesystem. + # The index tables will be prefixed with "index_". + - from: "2018-04-15" + store: cassandra + object_store: filesystem + schema: v11 + index: + period: 168h + prefix: index_ + + # Starting from 2020-6-15 we moved from filesystem to AWS S3 for storing the chunks. + - from: "2020-06-15" + store: cassandra + object_store: s3 + schema: v11 + index: + period: 168h + prefix: index_ + \ No newline at end of file diff --git a/docs/sources/configuration/examples/google-cloud-storage-config.yaml b/docs/sources/configuration/examples/google-cloud-storage-config.yaml new file mode 100644 index 000000000000..c76d90dac24e --- /dev/null +++ b/docs/sources/configuration/examples/google-cloud-storage-config.yaml @@ -0,0 +1,19 @@ +# This partial configuration uses GCS for chunk storage and uses BigTable for index storage + +schema_config: + configs: + - from: 2020-05-15 + store: bigtable + object_store: gcs + schema: v11 + index: + prefix: loki_index_ + period: 168h + +storage_config: + bigtable: + instance: BIGTABLE_INSTANCE + project: BIGTABLE_PROJECT + gcs: + bucket_name: GCS_BUCKET_NAME + \ No newline at end of file diff --git a/docs/sources/configuration/examples/s3-compatible-apis.yaml b/docs/sources/configuration/examples/s3-compatible-apis.yaml new file mode 100644 index 000000000000..6631cbb119a0 --- /dev/null +++ b/docs/sources/configuration/examples/s3-compatible-apis.yaml @@ -0,0 +1,18 @@ +# S3-compatible APIs such as Ceph Object Storage with an S3-compatible API, can be used. +# If the API supports path-style URLs rather than virtual hosted bucket addressing, +# configure the URL in `storage_config` with the custom endpoint + +schema_config: + configs: + - from: 2020-05-15 + store: aws + object_store: s3 + schema: v11 + index: + prefix: loki_ +storage_config: + aws: + s3: s3://access_key:secret_access_key@region/bucket_name + dynamodb: + dynamodb_url: dynamodb://access_key:secret_access_key@region + \ No newline at end of file diff --git a/docs/sources/configuration/examples/s3-expanded-config.yaml b/docs/sources/configuration/examples/s3-expanded-config.yaml new file mode 100644 index 000000000000..a407ade3c65c --- /dev/null +++ b/docs/sources/configuration/examples/s3-expanded-config.yaml @@ -0,0 +1,26 @@ +# S3 configuration supports an expanded configuration. +# Either an `s3` endpoint URL can be used, or an expanded configuration can be used. + +schema_config: + configs: + - from: 2020-05-15 + store: aws + object_store: s3 + schema: v11 + index: + prefix: loki_ +storage_config: + aws: + bucketnames: bucket_name1, bucket_name2 + endpoint: s3.endpoint.com + region: s3_region + access_key_id: s3_access_key_id + secret_access_key: s3_secret_access_key + insecure: false + sse_encryption: false + http_config: + idle_conn_timeout: 90s + response_header_timeout: 0s + insecure_skip_verify: false + s3forcepathstyle: true + \ No newline at end of file