From 342027c53022d98769474dda96a38622564863cc Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sat, 16 Jan 2016 16:20:00 -0800 Subject: [PATCH] Data source loading This allows the config loader to read "data" blocks from the config and turn them into DataSource objects. This just reads the data from the config file. It doesn't validate the data nor do anything useful with it. --- config/append.go | 8 ++ config/config.go | 15 +++ config/config_string.go | 70 ++++++++++++ config/loader_hcl.go | 104 ++++++++++++++++++ config/loader_test.go | 61 ++++++++++ config/merge.go | 17 +++ config/test-fixtures/basic.tf | 8 ++ config/test-fixtures/basic.tf.json | 11 ++ .../data-source-arity-mistake.tf | 3 + config/test-fixtures/dir-basic/one.tf | 4 + config/test-fixtures/dir-basic/two.tf | 4 + .../dir-override/foo_override.tf.json | 7 ++ config/test-fixtures/dir-override/one.tf | 5 + config/test-fixtures/dir-override/two.tf | 4 + 14 files changed, 321 insertions(+) create mode 100644 config/test-fixtures/data-source-arity-mistake.tf diff --git a/config/append.go b/config/append.go index bf13534e7c94..c43c368d15eb 100644 --- a/config/append.go +++ b/config/append.go @@ -57,6 +57,14 @@ func Append(c1, c2 *Config) (*Config, error) { c.ProviderConfigs = append(c.ProviderConfigs, c2.ProviderConfigs...) } + if len(c1.DataSources) > 0 || len(c2.DataSources) > 0 { + c.DataSources = make( + []*DataSource, + 0, len(c1.DataSources)+len(c2.DataSources)) + c.DataSources = append(c.DataSources, c1.DataSources...) + c.DataSources = append(c.DataSources, c2.DataSources...) + } + if len(c1.Resources) > 0 || len(c2.Resources) > 0 { c.Resources = make( []*Resource, diff --git a/config/config.go b/config/config.go index 6eeaf6dff3fa..cfadd58ae32d 100644 --- a/config/config.go +++ b/config/config.go @@ -772,6 +772,21 @@ func (c *ProviderConfig) mergerMerge(m merger) merger { return &result } +func (r *DataSource) mergerName() string { + return fmt.Sprintf("%s.%s", r.Type, r.Name) +} + +func (r *DataSource) mergerMerge(m merger) merger { + r2 := m.(*DataSource) + + result := *r + result.Name = r2.Name + result.Type = r2.Type + result.RawConfig = result.RawConfig.merge(r2.RawConfig) + + return &result +} + func (r *Resource) mergerName() string { return fmt.Sprintf("%s.%s", r.Type, r.Name) } diff --git a/config/config_string.go b/config/config_string.go index 4085d635f3f4..54373d26b313 100644 --- a/config/config_string.go +++ b/config/config_string.go @@ -170,6 +170,76 @@ func providerConfigsStr(pcs []*ProviderConfig) string { return strings.TrimSpace(result) } +// This helper turns a data sources field into a deterministic +// string value for comparison in tests. +func dataSourcesStr(rs []*DataSource) string { + result := "" + order := make([]int, 0, len(rs)) + ks := make([]string, 0, len(rs)) + mapping := make(map[string]int) + for i, r := range rs { + k := fmt.Sprintf("%s[%s]", r.Type, r.Name) + ks = append(ks, k) + mapping[k] = i + } + sort.Strings(ks) + for _, k := range ks { + order = append(order, mapping[k]) + } + + for _, i := range order { + r := rs[i] + result += fmt.Sprintf( + "%s[%s]\n", + r.Type, + r.Name) + + ks := make([]string, 0, len(r.RawConfig.Raw)) + for k, _ := range r.RawConfig.Raw { + ks = append(ks, k) + } + sort.Strings(ks) + + for _, k := range ks { + result += fmt.Sprintf(" %s\n", k) + } + + if len(r.DependsOn) > 0 { + result += fmt.Sprintf(" dependsOn\n") + for _, d := range r.DependsOn { + result += fmt.Sprintf(" %s\n", d) + } + } + + if len(r.RawConfig.Variables) > 0 { + result += fmt.Sprintf(" vars\n") + + ks := make([]string, 0, len(r.RawConfig.Variables)) + for k, _ := range r.RawConfig.Variables { + ks = append(ks, k) + } + sort.Strings(ks) + + for _, k := range ks { + rawV := r.RawConfig.Variables[k] + kind := "unknown" + str := rawV.FullKey() + + switch rawV.(type) { + case *ResourceVariable: + kind = "resource" + case *UserVariable: + kind = "user" + } + + result += fmt.Sprintf(" %s: %s\n", kind, str) + } + } + } + + return strings.TrimSpace(result) +} + // This helper turns a resources field into a deterministic // string value for comparison in tests. func resourcesStr(rs []*Resource) string { diff --git a/config/loader_hcl.go b/config/loader_hcl.go index 3757e8048332..7459b975c3b7 100644 --- a/config/loader_hcl.go +++ b/config/loader_hcl.go @@ -20,6 +20,7 @@ type hclConfigurable struct { func (t *hclConfigurable) Config() (*Config, error) { validKeys := map[string]struct{}{ "atlas": struct{}{}, + "data": struct{}{}, "module": struct{}{}, "output": struct{}{}, "provider": struct{}{}, @@ -112,6 +113,15 @@ func (t *hclConfigurable) Config() (*Config, error) { } } + // Build the data sources + if dataSources := list.Filter("data"); len(dataSources.Items) > 0 { + var err error + config.DataSources, err = loadDataSourcesHcl(dataSources) + if err != nil { + return nil, err + } + } + // Build the resources if resources := list.Filter("resource"); len(resources.Items) > 0 { var err error @@ -394,6 +404,100 @@ func loadProvidersHcl(list *ast.ObjectList) ([]*ProviderConfig, error) { return result, nil } +// Given a handle to a HCL object, this recurses into the structure +// and pulls out a list of data sources. +// +// The resulting data sources may not be unique, but each one +// represents exactly one data definition in the HCL configuration. +// We leave it up to another pass to merge them together. +func loadDataSourcesHcl(list *ast.ObjectList) ([]*DataSource, error) { + list = list.Children() + if len(list.Items) == 0 { + return nil, nil + } + + // Where all the results will go + var result []*DataSource + + // Now go over all the types and their children in order to get + // all of the actual resources. + for _, item := range list.Items { + if len(item.Keys) != 2 { + return nil, fmt.Errorf( + "position %s: 'data' must be followed by exactly two strings: a type and a name", + item.Pos()) + } + + t := item.Keys[0].Token.Value().(string) + k := item.Keys[1].Token.Value().(string) + + var listVal *ast.ObjectList + if ot, ok := item.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return nil, fmt.Errorf("data sources %s[%s]: should be an object", t, k) + } + + var config map[string]interface{} + if err := hcl.DecodeObject(&config, item.Val); err != nil { + return nil, fmt.Errorf( + "Error reading config for %s[%s]: %s", + t, + k, + err) + } + + // Remove the fields we handle specially + delete(config, "depends_on") + delete(config, "provider") + + rawConfig, err := NewRawConfig(config) + if err != nil { + return nil, fmt.Errorf( + "Error reading config for %s[%s]: %s", + t, + k, + err) + } + + // If we have depends fields, then add those in + var dependsOn []string + if o := listVal.Filter("depends_on"); len(o.Items) > 0 { + err := hcl.DecodeObject(&dependsOn, o.Items[0].Val) + if err != nil { + return nil, fmt.Errorf( + "Error reading depends_on for %s[%s]: %s", + t, + k, + err) + } + } + + // If we have a provider, then parse it out + var provider string + if o := listVal.Filter("provider"); len(o.Items) > 0 { + err := hcl.DecodeObject(&provider, o.Items[0].Val) + if err != nil { + return nil, fmt.Errorf( + "Error reading provider for %s[%s]: %s", + t, + k, + err) + } + } + + result = append(result, &DataSource{ + Name: k, + Type: t, + RawConfig: rawConfig, + Provider: provider, + DependsOn: dependsOn, + }) + } + + return result, nil +} + // Given a handle to a HCL object, this recurses into the structure // and pulls out a list of resources. // diff --git a/config/loader_test.go b/config/loader_test.go index 74ee7b9d55b9..786afc6169bf 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -65,6 +65,17 @@ func TestLoadFile_resourceArityMistake(t *testing.T) { } } +func TestLoadFile_dataSourceArityMistake(t *testing.T) { + _, err := LoadFile(filepath.Join(fixtureDir, "data-source-arity-mistake.tf")) + if err == nil { + t.Fatal("should have error") + } + expected := "Error loading test-fixtures/data-source-arity-mistake.tf: position 2:6: 'data' must be followed by exactly two strings: a type and a name" + if err.Error() != expected { + t.Fatalf("expected:\n%s\ngot:\n%s", expected, err) + } +} + func TestLoadFileWindowsLineEndings(t *testing.T) { testFile := filepath.Join(fixtureDir, "windows-line-endings.tf") @@ -169,6 +180,11 @@ func TestLoadFileBasic(t *testing.T) { t.Fatalf("bad:\n%s", actual) } + actual = dataSourcesStr(c.DataSources) + if actual != strings.TrimSpace(basicDataSourcesStr) { + t.Fatalf("bad:\n%s", actual) + } + actual = resourcesStr(c.Resources) if actual != strings.TrimSpace(basicResourcesStr) { t.Fatalf("bad:\n%s", actual) @@ -249,6 +265,11 @@ func TestLoadFileBasic_json(t *testing.T) { t.Fatalf("bad:\n%s", actual) } + actual = dataSourcesStr(c.DataSources) + if actual != strings.TrimSpace(basicDataSourcesStr) { + t.Fatalf("bad:\n%s", actual) + } + actual = resourcesStr(c.Resources) if actual != strings.TrimSpace(basicResourcesStr) { t.Fatalf("bad:\n%s", actual) @@ -314,6 +335,11 @@ func TestLoadJSONBasic(t *testing.T) { t.Fatalf("bad:\n%s", actual) } + actual = dataSourcesStr(c.DataSources) + if actual != strings.TrimSpace(basicDataSourcesStr) { + t.Fatalf("bad:\n%s", actual) + } + actual = resourcesStr(c.Resources) if actual != strings.TrimSpace(basicResourcesStr) { t.Fatalf("bad:\n%s", actual) @@ -373,6 +399,11 @@ func TestLoadDir_basic(t *testing.T) { t.Fatalf("bad:\n%s", actual) } + actual = dataSourcesStr(c.DataSources) + if actual != strings.TrimSpace(dirBasicDataSourcesStr) { + t.Fatalf("bad:\n%s", actual) + } + actual = resourcesStr(c.Resources) if actual != strings.TrimSpace(dirBasicResourcesStr) { t.Fatalf("bad:\n%s", actual) @@ -433,6 +464,11 @@ func TestLoadDir_override(t *testing.T) { t.Fatalf("bad:\n%s", actual) } + actual = dataSourcesStr(c.DataSources) + if actual != strings.TrimSpace(dirOverrideDataSourcesStr) { + t.Fatalf("bad:\n%s", actual) + } + actual = resourcesStr(c.Resources) if actual != strings.TrimSpace(dirOverrideResourcesStr) { t.Fatalf("bad:\n%s", actual) @@ -799,6 +835,14 @@ do user: var.foo ` +const basicDataSourcesStr = ` +do[depends] + dependsOn + data.do.simple +do[simple] + foo +` + const basicResourcesStr = ` aws_instance[db] (x1) VPC @@ -853,6 +897,14 @@ do user: var.foo ` +const dirBasicDataSourcesStr = ` +do[depends] + dependsOn + data.do.simple +do[simple] + foo +` + const dirBasicResourcesStr = ` aws_instance[db] (x1) security_groups @@ -890,6 +942,15 @@ do user: var.foo ` +const dirOverrideDataSourcesStr = ` +do[depends] + hello + dependsOn + data.do.simple +do[simple] + foo +` + const dirOverrideResourcesStr = ` aws_instance[db] (x1) ami diff --git a/config/merge.go b/config/merge.go index f72fdfa92093..29465031e7f4 100644 --- a/config/merge.go +++ b/config/merge.go @@ -93,6 +93,23 @@ func Merge(c1, c2 *Config) (*Config, error) { } } + // Data Sources + m1 = make([]merger, 0, len(c1.DataSources)) + m2 = make([]merger, 0, len(c2.DataSources)) + for _, v := range c1.DataSources { + m1 = append(m1, v) + } + for _, v := range c2.DataSources { + m2 = append(m2, v) + } + mresult = mergeSlice(m1, m2) + if len(mresult) > 0 { + c.DataSources = make([]*DataSource, len(mresult)) + for i, v := range mresult { + c.DataSources[i] = v.(*DataSource) + } + } + // Resources m1 = make([]merger, 0, len(c1.Resources)) m2 = make([]merger, 0, len(c2.Resources)) diff --git a/config/test-fixtures/basic.tf b/config/test-fixtures/basic.tf index 42c1d681b777..45314d54b571 100644 --- a/config/test-fixtures/basic.tf +++ b/config/test-fixtures/basic.tf @@ -24,6 +24,14 @@ provider "do" { api_key = "${var.foo}" } +data "do" "simple" { + foo = "baz" +} + +data "do" "depends" { + depends_on = ["data.do.simple"] +} + resource "aws_security_group" "firewall" { count = 5 } diff --git a/config/test-fixtures/basic.tf.json b/config/test-fixtures/basic.tf.json index ba8aa97492ad..be86d5de5a5d 100644 --- a/config/test-fixtures/basic.tf.json +++ b/config/test-fixtures/basic.tf.json @@ -26,6 +26,17 @@ } }, + "data": { + "do": { + "simple": { + "foo": "baz" + }, + "depends": { + "depends_on": ["data.do.simple"] + } + } + }, + "resource": { "aws_instance": { "db": { diff --git a/config/test-fixtures/data-source-arity-mistake.tf b/config/test-fixtures/data-source-arity-mistake.tf new file mode 100644 index 000000000000..5d579a9db62b --- /dev/null +++ b/config/test-fixtures/data-source-arity-mistake.tf @@ -0,0 +1,3 @@ +# I forgot the data source name! +data "null" { +} diff --git a/config/test-fixtures/dir-basic/one.tf b/config/test-fixtures/dir-basic/one.tf index 1e049a87f392..387c7b8d5723 100644 --- a/config/test-fixtures/dir-basic/one.tf +++ b/config/test-fixtures/dir-basic/one.tf @@ -8,6 +8,10 @@ provider "aws" { secret_key = "bar" } +data "do" "simple" { + foo = "baz" +} + resource "aws_instance" "db" { security_groups = "${aws_security_group.firewall.*.id}" } diff --git a/config/test-fixtures/dir-basic/two.tf b/config/test-fixtures/dir-basic/two.tf index acbb4f2f90a3..f64a2826387f 100644 --- a/config/test-fixtures/dir-basic/two.tf +++ b/config/test-fixtures/dir-basic/two.tf @@ -2,6 +2,10 @@ provider "do" { api_key = "${var.foo}" } +data "do" "depends" { + depends_on = ["data.do.simple"] +} + resource "aws_security_group" "firewall" { count = 5 } diff --git a/config/test-fixtures/dir-override/foo_override.tf.json b/config/test-fixtures/dir-override/foo_override.tf.json index 93c351b00a94..4a43f09fc528 100644 --- a/config/test-fixtures/dir-override/foo_override.tf.json +++ b/config/test-fixtures/dir-override/foo_override.tf.json @@ -1,4 +1,11 @@ { + "data": { + "do": { + "depends": { + "hello": "world" + } + } + }, "resource": { "aws_instance": { "web": { diff --git a/config/test-fixtures/dir-override/one.tf b/config/test-fixtures/dir-override/one.tf index 1e049a87f392..5ef1fe7191e1 100644 --- a/config/test-fixtures/dir-override/one.tf +++ b/config/test-fixtures/dir-override/one.tf @@ -8,6 +8,11 @@ provider "aws" { secret_key = "bar" } + +data "do" "simple" { + foo = "baz" +} + resource "aws_instance" "db" { security_groups = "${aws_security_group.firewall.*.id}" } diff --git a/config/test-fixtures/dir-override/two.tf b/config/test-fixtures/dir-override/two.tf index acbb4f2f90a3..f64a2826387f 100644 --- a/config/test-fixtures/dir-override/two.tf +++ b/config/test-fixtures/dir-override/two.tf @@ -2,6 +2,10 @@ provider "do" { api_key = "${var.foo}" } +data "do" "depends" { + depends_on = ["data.do.simple"] +} + resource "aws_security_group" "firewall" { count = 5 }