diff --git a/carbonsearch.example.yaml b/carbonsearch.example.yaml index 155a5e3..71c1035 100644 --- a/carbonsearch.example.yaml +++ b/carbonsearch.example.yaml @@ -38,7 +38,9 @@ text_index_service: "text" # 'servers' data source in 'servers-dc:us_east' associates tags with metrics # through the 'fqdn' join key. split_indexes: - fqdn: "servers" + fqdn: + - "servers" # like some asset management data source, or facts from the server itself + - "lb" # liveness in the loadbalancer, pool association for this host, etc. # ----consumers---- # adding a line to 'consumers' implies that carbonsearch should use this consumer. diff --git a/database/database.go b/database/database.go index ef3e4c6..8190989 100644 --- a/database/database.go +++ b/database/database.go @@ -292,7 +292,7 @@ func (db *Database) validateServiceIndexPairs(tags []string, givenIndex index.In func New( queryLimit, resultLimit int, fullIndexService, textIndexService string, - splitIndexConfig map[string]string, + splitIndexConfig map[string][]string, stats *util.Stats, ) *Database { serviceToIndex := make(map[string]index.Index) @@ -312,15 +312,23 @@ func New( writeBuffer := NewWriteBuffer(fullIndex.Name(), toc) splitIndexes := map[string]*split.Index{} - for joinKey, service := range splitIndexConfig { + for joinKey, services := range splitIndexConfig { index := split.NewIndex(joinKey) splitIndexes[joinKey] = index - serviceToIndex[service] = index err := writeBuffer.AddSplitIndex(joinKey) - toc.AddIndexServiceEntry("split", joinKey, service) if err != nil { panic(fmt.Sprintf("database: %v has already been loaded. This likely means the config file has %v listed multiple times", joinKey, joinKey)) } + + for _, service := range services { + _, ok := serviceToIndex[service] + if ok { + panic(fmt.Sprintf("database: service %v has already been attached to index %v. This likely means the config file has %q listed multiple times under %q in the 'split_indexes' section", service, joinKey, service, joinKey)) + + } + serviceToIndex[service] = index + toc.AddIndexServiceEntry("split", joinKey, service) + } } db := &Database{ diff --git a/database/database_test.go b/database/database_test.go index 22f1e3d..c6a8f31 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -23,8 +23,8 @@ var queryLimit = 10 var resultLimit = 10 var fullService = "custom" var textService = "text" -var splitIndexes = map[string]string{ - "fqdn": "servers", +var splitIndexes = map[string][]string{ + "fqdn": []string{"servers"}, } func TestFullQuery(t *testing.T) { @@ -302,8 +302,8 @@ func TestSplitQuery(t *testing.T) { } func TestTextQuery(t *testing.T) { - var unusedSplitIndexes = map[string]string{ - "foobar_unused_key": "foobar_unused_service", + var unusedSplitIndexes = map[string][]string{ + "foobar_unused_key": []string{"foobar_unused_service"}, } db := New(queryLimit, resultLimit, fullService, textService, unusedSplitIndexes, stats) diff --git a/main.go b/main.go index 52b8974..bc270a2 100644 --- a/main.go +++ b/main.go @@ -53,9 +53,9 @@ var Config = struct { GraphiteHost string `yaml:"graphite_host"` Consumers map[string]string `yaml:"consumers"` - FullIndexService string `yaml:"full_index_service"` - TextIndexService string `yaml:"text_index_service"` - SplitIndexes map[string]string `yaml:"split_indexes"` + FullIndexService string `yaml:"full_index_service"` + TextIndexService string `yaml:"text_index_service"` + SplitIndexes map[string][]string `yaml:"split_indexes"` }{ Port: 8070, diff --git a/scripts/populate_test_data.pl b/scripts/populate_test_data.pl index 3f575a7..f4038f3 100644 --- a/scripts/populate_test_data.pl +++ b/scripts/populate_test_data.pl @@ -19,7 +19,13 @@ my $port = $http_config->{port}; (my $endpoint = $http_config->{endpoint}) =~ s/^\///; my $splits = $main_config->{split_indexes}; -my %valid_services = map { $_ => 1 } sort values %$splits; + +my %valid_services; +for my $service_list (values %$splits) { + for my $service (@$service_list) { + $valid_services{$service} = 1; + } +} my %tag_soup = ( servers => {