cleanup boltdb files in queriers during startup/shutdown (#2558)

pkg/storage/stores/shipper/downloads/table.go

@@ -184,8 +184,14 @@ func (t *Table) Close() {
 	defer t.dbsMtx.Unlock()
 
 	for name, db := range t.dbs {
+		dbPath := db.boltdb.Path()
+
 		if err := db.boltdb.Close(); err != nil {
-			level.Error(util.Logger).Log("msg", fmt.Errorf("failed to close file %s for table %s", name, t.name))
+			level.Error(util.Logger).Log("msg", fmt.Sprintf("failed to close file %s for table %s", name, t.name), "err", err)
+		}
+
+		if err := os.Remove(dbPath); err != nil {
+			level.Error(util.Logger).Log("msg", fmt.Sprintf("failed to remove file %s for table %s", name, t.name), "err", err)
 		}
 	}
 

pkg/storage/stores/shipper/downloads/table_manager.go

@@ -3,6 +3,7 @@ package downloads
 import (
 	"context"
 	"fmt"
+	"os"
 	"sync"
 	"time"
 
@@ -37,6 +38,15 @@ type TableManager struct {
 }
 
 func NewTableManager(cfg Config, boltIndexClient BoltDBIndexClient, storageClient StorageClient, registerer prometheus.Registerer) (*TableManager, error) {
+	// cleanup existing directory and re-create it since we do not use existing files in it.
+	if err := os.RemoveAll(cfg.CacheDir); err != nil {
+		return nil, err
+	}
+
+	if err := chunk_util.EnsureDirectory(cfg.CacheDir); err != nil {
+		return nil, err
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	tm := &TableManager{
 		cfg:             cfg,