feat: allow variables to specify values in main.tf

apply/apply.go

@@ -14,13 +14,13 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
-	"golang.org/x/exp/slices"
 
 	v2 "github.com/chanzuckerberg/fogg/config/v2"
 	"github.com/chanzuckerberg/fogg/errs"
 	"github.com/chanzuckerberg/fogg/plan"
 	"github.com/chanzuckerberg/fogg/templates"
 	"github.com/chanzuckerberg/fogg/util"
+	"github.com/chanzuckerberg/go-misc/sets"
 	getter "github.com/hashicorp/go-getter"
 	"github.com/hashicorp/hcl2/hclwrite"
 	"github.com/hashicorp/terraform-config-inspect/tfconfig"
@@ -541,14 +541,88 @@ func applyTemplate(sourceFile io.Reader, commonTemplates fs.FS, dest afero.Fs, p
 	return t.Execute(writer, overrides)
 }
 
+func gatherTFModuleVariablesAndValues(component plan.Component, moduleConfig *tfconfig.Module) (map[string]string, error) {
+	// variables: [] -> add all required variables
+	// variables: -> add all required variables plus "test2" and set the value of "test" to be "value"
+	//   - test=value
+	//   - test2
+	variables := map[string]string{}
+	if component.Variables == nil {
+		for _, v := range moduleConfig.Variables {
+			variables[v.Name] = fmt.Sprintf("local.%s", v.Name)
+		}
+		return variables, nil
+	}
+
+	for _, v := range component.Variables {
+		keyValues := strings.SplitN(v, "=", 2)
+		if len(keyValues) < 1 {
+			return nil, fmt.Errorf("this should never happen; variables of a module_source must split into at least one substirng (ie 'key', 'key=value') ")
+		}
+		variables[keyValues[0]] = ""
+		if len(keyValues) == 2 {
+			variables[keyValues[0]] = keyValues[1]
+		}
+	}
+
+	for _, v := range moduleConfig.Variables {
+		// only add the required variables
+		if !v.Required {
+			continue
+		}
+
+		// don't override ones that the user has placed a value for
+		if vVal, ok := variables[v.Name]; ok && vVal != "" {
+			continue
+		}
+
+		variables[v.Name] = fmt.Sprintf("local.%s", v.Name)
+	}
+
+	// make sure all variables have a value
+	for k, v := range variables {
+		if v == "" {
+			variables[k] = fmt.Sprintf("local.%s", k)
+		}
+	}
+
+	// filter out non-existent variables on the module
+	ss := sets.NewStringSet()
+	for _, v := range moduleConfig.Variables {
+		ss.Add(v.Name)
+	}
+	for k := range variables {
+		if !ss.ContainsElement(k) {
+			delete(variables, k)
+		}
+	}
+
+	return variables, nil
+}
+
+func mapToSortedArray(m map[string]string) [][]string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+
+	keyVals := make([][]string, 0, len(m))
+	sort.Strings(keys)
+	for _, k := range keys {
+		keyVals = append(keyVals, []string{k, m[k]})
+	}
+
+	return keyVals
+}
+
 // This should really be part of the plan stage, not apply. But going to
 // leave it here for now and re-think it when we make this mechanism
 // general purpose.
 type moduleData struct {
 	ModuleName      string
 	ModuleSource    string
 	ProviderAliases map[string]string
-	Variables       []string
+	Variables       [][]string
 	Outputs         []*tfconfig.Output
 }
 
@@ -570,19 +644,11 @@ func applyModuleInvocation(
 		return errs.WrapUser(e, "could not download or parse module")
 	}
 
-	// This should really be part of the plan stage, not apply. But going to
-	// leave it here for now and re-think it when we make this mechanism
-	// general purpose.
-	addAll := component.Variables == nil
-	for _, v := range moduleConfig.Variables {
-		if addAll {
-			component.Variables = append(component.Variables, v.Name)
-		} else {
-			if v.Required && !slices.Contains(component.Variables, v.Name) {
-				component.Variables = append(component.Variables, v.Name)
-			}
-		}
+	variablesValues, err := gatherTFModuleVariablesAndValues(component, moduleConfig)
+	if err != nil {
+		return err
 	}
+
 	sort.Strings(component.Variables)
 
 	outputs := make([]*tfconfig.Output, 0)
@@ -613,7 +679,7 @@ func applyModuleInvocation(
 		ModuleName:      moduleName,
 		ModuleSource:    moduleAddressForSource,
 		ProviderAliases: component.ProviderAliases,
-		Variables:       component.Variables,
+		Variables:       mapToSortedArray(variablesValues),
 		Outputs:         outputs,
 	}
 	e = applyTemplate(

templates/templates/module-invocation/main.tf.tmpl

@@ -3,8 +3,8 @@
 
 module "{{.ModuleName}}" {
   source = "{{.ModuleSource}}"
-  {{range .Variables -}}
-    {{.}} = local.{{.}}
+  {{range $i, $kv := .Variables -}}
+    {{index $kv 0}} = {{index $kv 1}}
   {{ end}}
 
   {{ if .ProviderAliases -}}

testdata/tfe_config/fogg.yml

@@ -32,7 +32,11 @@ tfe:
   gh_repo: fogg
   tfe_org: si.prod.tfe.czi.technology
   module_source:  github.com/chanzuckerberg/cztack//aws-aurora-postgres?ref=main
-  variables: []
+  variables: 
+    - env=var.tags.env
+    - owner=var.tags.owner
+    - project
+    - blah # should be filtered out
   extra_vars:
     TFE_AWS_ACCESS_KEY_ID: ""
     TFE_AWS_SECRET_ACCESS_KEY: ""

testdata/tfe_config/terraform/tfe/main.tf

@@ -7,8 +7,8 @@ module "aws-aurora-postgres" {
   database_password     = local.database_password
   database_subnet_group = local.database_subnet_group
   database_username     = local.database_username
-  env                   = local.env
-  owner                 = local.owner
+  env                   = var.tags.env
+  owner                 = var.tags.owner
   project               = local.project
   service               = local.service
   vpc_id                = local.vpc_id