Data source loading

This allows the config loader to read "data" blocks from the config and
turn them into DataSource objects.

This just reads the data from the config file. It doesn't validate the
data nor do anything useful with it.

config/append.go

@@ -57,6 +57,14 @@ func Append(c1, c2 *Config) (*Config, error) {
 		c.ProviderConfigs = append(c.ProviderConfigs, c2.ProviderConfigs...)
 	}
 
+	if len(c1.DataSources) > 0 || len(c2.DataSources) > 0 {
+		c.DataSources = make(
+			[]*DataSource,
+			0, len(c1.DataSources)+len(c2.DataSources))
+		c.DataSources = append(c.DataSources, c1.DataSources...)
+		c.DataSources = append(c.DataSources, c2.DataSources...)
+	}
+
 	if len(c1.Resources) > 0 || len(c2.Resources) > 0 {
 		c.Resources = make(
 			[]*Resource,

config/config.go

@@ -772,6 +772,21 @@ func (c *ProviderConfig) mergerMerge(m merger) merger {
 	return &result
 }
 
+func (r *DataSource) mergerName() string {
+	return fmt.Sprintf("%s.%s", r.Type, r.Name)
+}
+
+func (r *DataSource) mergerMerge(m merger) merger {
+	r2 := m.(*DataSource)
+
+	result := *r
+	result.Name = r2.Name
+	result.Type = r2.Type
+	result.RawConfig = result.RawConfig.merge(r2.RawConfig)
+
+	return &result
+}
+
 func (r *Resource) mergerName() string {
 	return fmt.Sprintf("%s.%s", r.Type, r.Name)
 }

config/config_string.go

@@ -170,6 +170,76 @@ func providerConfigsStr(pcs []*ProviderConfig) string {
 	return strings.TrimSpace(result)
 }
 
+// This helper turns a data sources field into a deterministic
+// string value for comparison in tests.
+func dataSourcesStr(rs []*DataSource) string {
+	result := ""
+	order := make([]int, 0, len(rs))
+	ks := make([]string, 0, len(rs))
+	mapping := make(map[string]int)
+	for i, r := range rs {
+		k := fmt.Sprintf("%s[%s]", r.Type, r.Name)
+		ks = append(ks, k)
+		mapping[k] = i
+	}
+	sort.Strings(ks)
+	for _, k := range ks {
+		order = append(order, mapping[k])
+	}
+
+	for _, i := range order {
+		r := rs[i]
+		result += fmt.Sprintf(
+			"%s[%s]\n",
+			r.Type,
+			r.Name)
+
+		ks := make([]string, 0, len(r.RawConfig.Raw))
+		for k, _ := range r.RawConfig.Raw {
+			ks = append(ks, k)
+		}
+		sort.Strings(ks)
+
+		for _, k := range ks {
+			result += fmt.Sprintf("  %s\n", k)
+		}
+
+		if len(r.DependsOn) > 0 {
+			result += fmt.Sprintf("  dependsOn\n")
+			for _, d := range r.DependsOn {
+				result += fmt.Sprintf("    %s\n", d)
+			}
+		}
+
+		if len(r.RawConfig.Variables) > 0 {
+			result += fmt.Sprintf("  vars\n")
+
+			ks := make([]string, 0, len(r.RawConfig.Variables))
+			for k, _ := range r.RawConfig.Variables {
+				ks = append(ks, k)
+			}
+			sort.Strings(ks)
+
+			for _, k := range ks {
+				rawV := r.RawConfig.Variables[k]
+				kind := "unknown"
+				str := rawV.FullKey()
+
+				switch rawV.(type) {
+				case *ResourceVariable:
+					kind = "resource"
+				case *UserVariable:
+					kind = "user"
+				}
+
+				result += fmt.Sprintf("    %s: %s\n", kind, str)
+			}
+		}
+	}
+
+	return strings.TrimSpace(result)
+}
+
 // This helper turns a resources field into a deterministic
 // string value for comparison in tests.
 func resourcesStr(rs []*Resource) string {

config/loader_hcl.go

@@ -20,6 +20,7 @@ type hclConfigurable struct {
 func (t *hclConfigurable) Config() (*Config, error) {
 	validKeys := map[string]struct{}{
 		"atlas":    struct{}{},
+		"data":     struct{}{},
 		"module":   struct{}{},
 		"output":   struct{}{},
 		"provider": struct{}{},
@@ -112,6 +113,15 @@ func (t *hclConfigurable) Config() (*Config, error) {
 		}
 	}
 
+	// Build the data sources
+	if dataSources := list.Filter("data"); len(dataSources.Items) > 0 {
+		var err error
+		config.DataSources, err = loadDataSourcesHcl(dataSources)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Build the resources
 	if resources := list.Filter("resource"); len(resources.Items) > 0 {
 		var err error
@@ -394,6 +404,100 @@ func loadProvidersHcl(list *ast.ObjectList) ([]*ProviderConfig, error) {
 	return result, nil
 }
 
+// Given a handle to a HCL object, this recurses into the structure
+// and pulls out a list of data sources.
+//
+// The resulting data sources may not be unique, but each one
+// represents exactly one data definition in the HCL configuration.
+// We leave it up to another pass to merge them together.
+func loadDataSourcesHcl(list *ast.ObjectList) ([]*DataSource, error) {
+	list = list.Children()
+	if len(list.Items) == 0 {
+		return nil, nil
+	}
+
+	// Where all the results will go
+	var result []*DataSource
+
+	// Now go over all the types and their children in order to get
+	// all of the actual resources.
+	for _, item := range list.Items {
+		if len(item.Keys) != 2 {
+			return nil, fmt.Errorf(
+				"position %s: 'data' must be followed by exactly two strings: a type and a name",
+				item.Pos())
+		}
+
+		t := item.Keys[0].Token.Value().(string)
+		k := item.Keys[1].Token.Value().(string)
+
+		var listVal *ast.ObjectList
+		if ot, ok := item.Val.(*ast.ObjectType); ok {
+			listVal = ot.List
+		} else {
+			return nil, fmt.Errorf("data sources %s[%s]: should be an object", t, k)
+		}
+
+		var config map[string]interface{}
+		if err := hcl.DecodeObject(&config, item.Val); err != nil {
+			return nil, fmt.Errorf(
+				"Error reading config for %s[%s]: %s",
+				t,
+				k,
+				err)
+		}
+
+		// Remove the fields we handle specially
+		delete(config, "depends_on")
+		delete(config, "provider")
+
+		rawConfig, err := NewRawConfig(config)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"Error reading config for %s[%s]: %s",
+				t,
+				k,
+				err)
+		}
+
+		// If we have depends fields, then add those in
+		var dependsOn []string
+		if o := listVal.Filter("depends_on"); len(o.Items) > 0 {
+			err := hcl.DecodeObject(&dependsOn, o.Items[0].Val)
+			if err != nil {
+				return nil, fmt.Errorf(
+					"Error reading depends_on for %s[%s]: %s",
+					t,
+					k,
+					err)
+			}
+		}
+
+		// If we have a provider, then parse it out
+		var provider string
+		if o := listVal.Filter("provider"); len(o.Items) > 0 {
+			err := hcl.DecodeObject(&provider, o.Items[0].Val)
+			if err != nil {
+				return nil, fmt.Errorf(
+					"Error reading provider for %s[%s]: %s",
+					t,
+					k,
+					err)
+			}
+		}
+
+		result = append(result, &DataSource{
+			Name:      k,
+			Type:      t,
+			RawConfig: rawConfig,
+			Provider:  provider,
+			DependsOn: dependsOn,
+		})
+	}
+
+	return result, nil
+}
+
 // Given a handle to a HCL object, this recurses into the structure
 // and pulls out a list of resources.
 //

config/loader_test.go

@@ -65,6 +65,17 @@ func TestLoadFile_resourceArityMistake(t *testing.T) {
 	}
 }
 
+func TestLoadFile_dataSourceArityMistake(t *testing.T) {
+	_, err := LoadFile(filepath.Join(fixtureDir, "data-source-arity-mistake.tf"))
+	if err == nil {
+		t.Fatal("should have error")
+	}
+	expected := "Error loading test-fixtures/data-source-arity-mistake.tf: position 2:6: 'data' must be followed by exactly two strings: a type and a name"
+	if err.Error() != expected {
+		t.Fatalf("expected:\n%s\ngot:\n%s", expected, err)
+	}
+}
+
 func TestLoadFileWindowsLineEndings(t *testing.T) {
 	testFile := filepath.Join(fixtureDir, "windows-line-endings.tf")
 
@@ -169,6 +180,11 @@ func TestLoadFileBasic(t *testing.T) {
 		t.Fatalf("bad:\n%s", actual)
 	}
 
+	actual = dataSourcesStr(c.DataSources)
+	if actual != strings.TrimSpace(basicDataSourcesStr) {
+		t.Fatalf("bad:\n%s", actual)
+	}
+
 	actual = resourcesStr(c.Resources)
 	if actual != strings.TrimSpace(basicResourcesStr) {
 		t.Fatalf("bad:\n%s", actual)
@@ -249,6 +265,11 @@ func TestLoadFileBasic_json(t *testing.T) {
 		t.Fatalf("bad:\n%s", actual)
 	}
 
+	actual = dataSourcesStr(c.DataSources)
+	if actual != strings.TrimSpace(basicDataSourcesStr) {
+		t.Fatalf("bad:\n%s", actual)
+	}
+
 	actual = resourcesStr(c.Resources)
 	if actual != strings.TrimSpace(basicResourcesStr) {
 		t.Fatalf("bad:\n%s", actual)
@@ -314,6 +335,11 @@ func TestLoadJSONBasic(t *testing.T) {
 		t.Fatalf("bad:\n%s", actual)
 	}
 
+	actual = dataSourcesStr(c.DataSources)
+	if actual != strings.TrimSpace(basicDataSourcesStr) {
+		t.Fatalf("bad:\n%s", actual)
+	}
+
 	actual = resourcesStr(c.Resources)
 	if actual != strings.TrimSpace(basicResourcesStr) {
 		t.Fatalf("bad:\n%s", actual)
@@ -373,6 +399,11 @@ func TestLoadDir_basic(t *testing.T) {
 		t.Fatalf("bad:\n%s", actual)
 	}
 
+	actual = dataSourcesStr(c.DataSources)
+	if actual != strings.TrimSpace(dirBasicDataSourcesStr) {
+		t.Fatalf("bad:\n%s", actual)
+	}
+
 	actual = resourcesStr(c.Resources)
 	if actual != strings.TrimSpace(dirBasicResourcesStr) {
 		t.Fatalf("bad:\n%s", actual)
@@ -433,6 +464,11 @@ func TestLoadDir_override(t *testing.T) {
 		t.Fatalf("bad:\n%s", actual)
 	}
 
+	actual = dataSourcesStr(c.DataSources)
+	if actual != strings.TrimSpace(dirOverrideDataSourcesStr) {
+		t.Fatalf("bad:\n%s", actual)
+	}
+
 	actual = resourcesStr(c.Resources)
 	if actual != strings.TrimSpace(dirOverrideResourcesStr) {
 		t.Fatalf("bad:\n%s", actual)
@@ -799,6 +835,14 @@ do
     user: var.foo
 `
 
+const basicDataSourcesStr = `
+do[depends]
+  dependsOn
+    data.do.simple
+do[simple]
+  foo
+`
+
 const basicResourcesStr = `
 aws_instance[db] (x1)
   VPC
@@ -853,6 +897,14 @@ do
     user: var.foo
 `
 
+const dirBasicDataSourcesStr = `
+do[depends]
+  dependsOn
+    data.do.simple
+do[simple]
+  foo
+`
+
 const dirBasicResourcesStr = `
 aws_instance[db] (x1)
   security_groups
@@ -890,6 +942,15 @@ do
     user: var.foo
 `
 
+const dirOverrideDataSourcesStr = `
+do[depends]
+  hello
+  dependsOn
+    data.do.simple
+do[simple]
+  foo
+`
+
 const dirOverrideResourcesStr = `
 aws_instance[db] (x1)
   ami

config/merge.go

@@ -93,6 +93,23 @@ func Merge(c1, c2 *Config) (*Config, error) {
 		}
 	}
 
+	// Data Sources
+	m1 = make([]merger, 0, len(c1.DataSources))
+	m2 = make([]merger, 0, len(c2.DataSources))
+	for _, v := range c1.DataSources {
+		m1 = append(m1, v)
+	}
+	for _, v := range c2.DataSources {
+		m2 = append(m2, v)
+	}
+	mresult = mergeSlice(m1, m2)
+	if len(mresult) > 0 {
+		c.DataSources = make([]*DataSource, len(mresult))
+		for i, v := range mresult {
+			c.DataSources[i] = v.(*DataSource)
+		}
+	}
+
 	// Resources
 	m1 = make([]merger, 0, len(c1.Resources))
 	m2 = make([]merger, 0, len(c2.Resources))