config: change split_indexes to take a list of services

indexes (like 'fqdn') should be listed once, but each index can serve
all the tag prefixes which use this same join key (for example,
hostname). so attributes from asset management ('servers-dc:us_east'),
and attributes from the load balancer ('lb-pool:frontend') can both live
in the 'fqdn' index, because both of them associate metadata with
metrics through the hostname of the box producing metrics.

carbonsearch.example.yaml

@@ -38,7 +38,9 @@ text_index_service: "text"
 # 'servers' data source in 'servers-dc:us_east' associates tags with metrics
 # through the 'fqdn' join key.
 split_indexes:
-    fqdn: "servers"
+    fqdn:
+        - "servers" # like some asset management data source, or facts from the server itself
+        - "lb" # liveness in the loadbalancer, pool association for this host, etc.
 
 # ----consumers----
 # adding a line to 'consumers' implies that carbonsearch should use this consumer.

database/database.go

@@ -292,7 +292,7 @@ func (db *Database) validateServiceIndexPairs(tags []string, givenIndex index.In
 func New(
 	queryLimit, resultLimit int,
 	fullIndexService, textIndexService string,
-	splitIndexConfig map[string]string,
+	splitIndexConfig map[string][]string,
 	stats *util.Stats,
 ) *Database {
 	serviceToIndex := make(map[string]index.Index)
@@ -312,15 +312,23 @@ func New(
 
 	writeBuffer := NewWriteBuffer(fullIndex.Name(), toc)
 	splitIndexes := map[string]*split.Index{}
-	for joinKey, service := range splitIndexConfig {
+	for joinKey, services := range splitIndexConfig {
 		index := split.NewIndex(joinKey)
 		splitIndexes[joinKey] = index
-		serviceToIndex[service] = index
 		err := writeBuffer.AddSplitIndex(joinKey)
-		toc.AddIndexServiceEntry("split", joinKey, service)
 		if err != nil {
 			panic(fmt.Sprintf("database: %v has already been loaded. This likely means the config file has %v listed multiple times", joinKey, joinKey))
 		}
+
+		for _, service := range services {
+			_, ok := serviceToIndex[service]
+			if ok {
+				panic(fmt.Sprintf("database: service %v has already been attached to index %v. This likely means the config file has %q listed multiple times under %q in the 'split_indexes' section", service, joinKey, service, joinKey))
+
+			}
+			serviceToIndex[service] = index
+			toc.AddIndexServiceEntry("split", joinKey, service)
+		}
 	}
 
 	db := &Database{

database/database_test.go

@@ -23,8 +23,8 @@ var queryLimit = 10
 var resultLimit = 10
 var fullService = "custom"
 var textService = "text"
-var splitIndexes = map[string]string{
-	"fqdn": "servers",
+var splitIndexes = map[string][]string{
+	"fqdn": []string{"servers"},
 }
 
 func TestFullQuery(t *testing.T) {
@@ -302,8 +302,8 @@ func TestSplitQuery(t *testing.T) {
 }
 
 func TestTextQuery(t *testing.T) {
-	var unusedSplitIndexes = map[string]string{
-		"foobar_unused_key": "foobar_unused_service",
+	var unusedSplitIndexes = map[string][]string{
+		"foobar_unused_key": []string{"foobar_unused_service"},
 	}
 
 	db := New(queryLimit, resultLimit, fullService, textService, unusedSplitIndexes, stats)

main.go

@@ -53,9 +53,9 @@ var Config = struct {
 	GraphiteHost      string            `yaml:"graphite_host"`
 	Consumers         map[string]string `yaml:"consumers"`
 
-	FullIndexService string            `yaml:"full_index_service"`
-	TextIndexService string            `yaml:"text_index_service"`
-	SplitIndexes     map[string]string `yaml:"split_indexes"`
+	FullIndexService string              `yaml:"full_index_service"`
+	TextIndexService string              `yaml:"text_index_service"`
+	SplitIndexes     map[string][]string `yaml:"split_indexes"`
 }{
 	Port: 8070,
 

scripts/populate_test_data.pl

@@ -19,7 +19,13 @@
 my $port = $http_config->{port};
 (my $endpoint = $http_config->{endpoint}) =~ s/^\///;
 my $splits = $main_config->{split_indexes};
-my %valid_services = map { $_ => 1 } sort values %$splits;
+
+my %valid_services;
+for my $service_list (values %$splits) {
+    for my $service (@$service_list) {
+        $valid_services{$service} = 1;
+    }
+}
 
 my %tag_soup = (
     servers => {