Loki migrate-tool

.gitignore

@@ -17,6 +17,7 @@ cmd/docker-driver/docker-driver
 cmd/loki-canary/loki-canary
 cmd/fluent-bit/out_loki.so
 cmd/fluent-bit/out_loki.h
+cmd/migrate/migrate
 /loki
 /promtail
 /logcli

Makefile

@@ -6,6 +6,7 @@
 .PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image
 .PHONY: bigtable-backup, push-bigtable-backup
 .PHONY: benchmark-store, drone, check-mod
+.PHONY: migrate migrate-image
 
 SHELL = /usr/bin/env bash
 
@@ -225,6 +226,16 @@ cmd/promtail/promtail-debug: $(APP_GO_FILES) pkg/promtail/server/ui/assets_vfsda
 	CGO_ENABLED=$(PROMTAIL_CGO) go build $(PROMTAIL_DEBUG_GO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
+###############
+# Migrate #
+###############
+
+migrate: cmd/migrate/migrate
+
+cmd/migrate/migrate: $(APP_GO_FILES) cmd/migrate/main.go
+	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
+	$(NETGO_CHECK)
+
 #############
 # Releasing #
 #############
@@ -273,6 +284,7 @@ clean:
 	rm -rf dist/
 	rm -rf cmd/fluent-bit/out_grafana_loki.h
 	rm -rf cmd/fluent-bit/out_grafana_loki.so
+    rm -rf cmd/migrate/migrate
 	go clean $(MOD_FLAG) ./...
 
 #########
@@ -502,6 +514,11 @@ loki-querytee-image-cross:
 loki-querytee-push: loki-querytee-image-cross
 	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/loki-querytee:$(IMAGE_TAG)
 
+# migrate-image
+migrate-image:
+	$(SUDO) docker build -t $(IMAGE_PREFIX)/loki-migrate:$(IMAGE_TAG) -f cmd/migrate/Dockerfile .
+
+
 # build-image (only amd64)
 build-image: OCI_PLATFORMS=
 build-image:

cmd/migrate/Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:1.15.3 as build
+COPY . /src/loki
+WORKDIR /src/loki
+RUN make clean && make BUILD_IN_CONTAINER=false migrate
+
+FROM alpine:3.9
+RUN apk add --update --no-cache ca-certificates
+COPY --from=build /src/loki/cmd/migrate/migrate /usr/bin/migrate
+#ENTRYPOINT [ "/usr/bin/migrate" ]
+CMD tail -f /dev/null

cmd/migrate/README.md

@@ -0,0 +1,56 @@
+# Loki Migrate Tool
+
+**WARNING: THIS TOOL IS NOT WELL TESTED, ALWAYS MAKE BACKUPS AND TEST ON LESS IMPORTANT DATA FIRST!**
+
+This is sort of a bare minimum code hooked directly into the store interfaces within Loki.
+
+Two stores are created, a source store and destination (abbreviated dest) store.
+
+Chunks are queried from the source store and written to the dest store, new index entries are created in the dest store as well.
+
+This _should_ handle schema changes and different schemas on both the source and dest store.
+
+You should be able to:
+
+* Migrate between clusters
+* Change tenant ID during migration
+* Migrate data between schemas
+
+All data is read and re-written (even when migrating within the same cluster). There are really no optimizations in this code for performance and there are much faster ways to move data depending on what you want to change.
+
+This is simple and because it uses the storage interfaces, should be complete and should stay working, but it's not optimized to be fast.
+
+There is however some parallelism built in and there are a few flags to tune this, `migrate -help` for more info
+
+This does not remove any source data, it only reads existing source data and writes to the destination.
+
+## Usage
+
+Build with
+
+```
+make migrate
+```
+
+or
+
+```
+make migrate-image
+```
+
+The docker image currently runs and doesn't do anything, it's intended you exec into the container and run commands manually.
+
+
+### Examples
+
+Migrate between clusters
+
+```
+migrate -source.config.file=/etc/loki-us-west1/config/config.yaml -dest.config.file=/etc/loki-us-central1/config/config.yaml -source.tenant=2289 -dest.tenant=2289 -from=2020-06-16T14:00:00-00:00 -to=2020-07-01T00:00:00-00:00
+```
+
+Migrate tenant ID within a cluster
+
+```
+migrate -source.config.file=/etc/loki-us-west1/config/config.yaml -dest.config.file=/etc/loki-us-west1/config/config.yaml -source.tenant=fake -dest.tenant=1 -from=2020-06-16T14:00:00-00:00 -to=2020-07-01T00:00:00-00:00
+```