cloud-env via parsing module

Gopkg.toml

@@ -32,4 +32,8 @@
 
 [[constraint]]
   name = "github.com/pkg/errors"
-  version = "0.8.0"
+  branch = "master"
+
+[[constraint]]
+  name = "github.com/hashicorp/terraform"
+  version = "0.11.7"

Makefile

@@ -10,7 +10,7 @@ lint:
 	gometalinter --vendor --fast ./...
 
 lint-slow:
-	gometalinter --vendor ./...
+	gometalinter --vendor --deadline 120s ./...
 
 build:
 	go build ${LDFLAGS} .

apply/apply.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -14,6 +15,10 @@ import (
 	"github.com/chanzuckerberg/fogg/util"
 	"github.com/gobuffalo/packr"
 	"github.com/hashicorp/hcl/hcl/printer"
+	"github.com/hashicorp/terraform/config"
+	"github.com/hashicorp/terraform/config/module"
+	"github.com/hashicorp/terraform/svchost/auth"
+	"github.com/hashicorp/terraform/svchost/disco"
 	"github.com/pkg/errors"
 	"github.com/spf13/afero"
 )
@@ -107,7 +112,7 @@ func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBox *packr
 			return errors.Wrap(e, "unable to apply templates to env")
 		}
 		for component, componentPlan := range envPlan.Components {
-			path := fmt.Sprintf("%s/envs/%s/%s", rootPath, env, component)
+			path = fmt.Sprintf("%s/envs/%s/%s", rootPath, env, component)
 			e = fs.MkdirAll(path, 0755)
 			if e != nil {
 				return errors.Wrap(e, "unable to make directories for component")
@@ -117,6 +122,13 @@ func applyEnvs(fs afero.Fs, p *plan.Plan, envBox *packr.Box, componentBox *packr
 				return errors.Wrap(e, "unable to apply templates for component")
 			}
 		}
+		path = filepath.Join(rootPath, "envs", env, "cloud-env")
+		if envPlan.Type == "aws" {
+			e := applyModule(fs, path, "git@github.com:chanzuckerberg/shared-infra//terraform/modules/aws-env", templates.Templates.ModuleInvocation)
+			if e != nil {
+				return errors.Wrap(e, "unable to apply module")
+			}
+		}
 	}
 	return nil
 }
@@ -229,6 +241,82 @@ func applyTemplate(sourceFile io.Reader, dest afero.Fs, path string, overrides i
 	return t.Execute(writer, overrides)
 }
 
+func downloadModule(dir, mod string) error {
+	disco := disco.NewDisco()
+	s := module.NewStorage(dir, disco, auth.NoCredentials)
+
+	return s.GetModule(dir, mod)
+}
+
+func downloadAndParseModule(mod string) (*config.Config, error) {
+	dir, e := ioutil.TempDir("", "fogg")
+	if e != nil {
+		return nil, errors.Wrap(e, "unable to create tempdir")
+	}
+	e = downloadModule(dir, mod)
+	if e != nil {
+		return nil, errors.Wrap(e, "unable to download module")
+	}
+	return config.LoadDir(dir)
+}
+
+// This should really be part of the plan stage, not apply. But going to
+// leave it here for now and re-think it when we make this mechanism
+// general purpose.
+type moduleData struct {
+	ModuleName   string
+	ModuleSource string
+	Variables    []string
+	Outputs      []string
+}
+
+func applyModule(fs afero.Fs, path, mod string, box packr.Box) error {
+	e := fs.MkdirAll(path, 0755)
+	if e != nil {
+		return errors.Wrapf(e, "couldn't create %s directory", path)
+	}
+
+	c, e := downloadAndParseModule(mod)
+	if e != nil {
+		return errors.Wrap(e, "could not download or parse module")
+	}
+
+	// This should really be part of the plan stage, not apply. But going to
+	// leave it here for now and re-think it when we make this mechanism
+	// general purpose.
+	variables := make([]string, 0)
+	for _, v := range c.Variables {
+		variables = append(variables, v.Name)
+	}
+	outputs := make([]string, 0)
+	for _, o := range c.Outputs {
+		outputs = append(outputs, o.Name)
+	}
+	moduleName := filepath.Base(mod)
+
+	f, e := box.Open("main.tf.tmpl")
+	if e != nil {
+		return errors.Wrap(e, "could not open template file")
+	}
+
+	e = applyTemplate(f, fs, filepath.Join(path, "main.tf"), &moduleData{moduleName, mod, variables, outputs})
+	if e != nil {
+		return e
+	}
+
+	f, e = box.Open("outputs.tf.tmpl")
+	if e != nil {
+		return errors.Wrap(e, "could not open template file")
+	}
+
+	e = applyTemplate(f, fs, filepath.Join(path, "outputs.tf"), &moduleData{moduleName, mod, variables, outputs})
+	if e != nil {
+		return e
+	}
+
+	return nil
+}
+
 func getTargetPath(path string, siccMode bool) string {
 	target := path
 	extension := filepath.Ext(path)

apply/apply_test.go

@@ -149,6 +149,60 @@ func TestApplySmokeTest(t *testing.T) {
 	assert.Nil(t, e)
 }
 
+func TestDownloadModule(t *testing.T) {
+	dir, e := ioutil.TempDir("", "fogg")
+	assert.Nil(t, e)
+
+	e = downloadModule(dir, "./test-module")
+	assert.Nil(t, e)
+	// TODO more asserts
+}
+
+func TestDownloadAndParseModule(t *testing.T) {
+	c, e := downloadAndParseModule("./test-module")
+	assert.Nil(t, e)
+	assert.NotNil(t, c)
+	assert.Len(t, c.Variables, 2)
+	assert.Len(t, c.Outputs, 2)
+}
+
+func TestApplyModule(t *testing.T) {
+	fs := afero.NewMemMapFs()
+
+	e := applyModule(fs, "mymodule", "./test-module", templates.Templates.ModuleInvocation)
+	assert.Nil(t, e)
+
+	s, e := fs.Stat("mymodule")
+	assert.Nil(t, e)
+	assert.True(t, s.IsDir())
+
+	_, e = fs.Stat("mymodule/main.tf")
+	assert.Nil(t, e)
+	r, e := afero.ReadFile(fs, "mymodule/main.tf")
+	assert.Nil(t, e)
+	expected := `module "test-module" {
+  source = "./test-module"
+  foo = "${var.foo}"
+  bar = "${var.bar}"
+  
+}`
+	assert.Equal(t, expected, string(r))
+
+	_, e = fs.Stat("mymodule/outputs.tf")
+	assert.Nil(t, e)
+	r, e = afero.ReadFile(fs, "mymodule/outputs.tf")
+	assert.Nil(t, e)
+	expected = `output "foo" {
+  value = "${module.test-module.foo}"
+}
+
+output "bar" {
+  value = "${module.test-module.bar}"
+}
+
+`
+	assert.Equal(t, expected, string(r))
+}
 func TestGetTargetPath(t *testing.T) {
 	data := []struct {
 		source  string

apply/test-module/outputs.tf

@@ -0,0 +1,7 @@
+output "foo" {
+  value = "${var.foo}"
+}
+
+output "bar" {
+  value = "${var.bar}"
+}

apply/test-module/variables.tf

@@ -0,0 +1,2 @@
+variable "foo" {}
+variable "bar" {}

config/config.go

@@ -18,6 +18,7 @@ type defaults struct {
 	InfraBucket        string   `json:"infra_s3_bucket"`
 	Owner              string   `json:"owner"`
 	Project            string   `json:"project"`
+	SharedInfraVersion string   `json:"shared_infra_version"`
 	TerraformVersion   string   `json:"terraform_version"`
 }
 
@@ -45,6 +46,7 @@ type Env struct {
 	Owner              *string  `json:"owner"`
 	Project            *string  `json:"project"`
 	TerraformVersion   *string  `json:"terraform_version"`
+	Type               *string  `json:"type"`
 
 	Components map[string]*Component `json:"components"`
 }
@@ -59,6 +61,7 @@ type Component struct {
 	InfraBucket        *string  `json:"infra_s3_bucket"`
 	Owner              *string  `json:"owner"`
 	Project            *string  `json:"project"`
+	SharedInfraVersion *string  `json:"shared_infra_version"`
 	TerraformVersion   *string  `json:"terraform_version"`
 }
 

plan/plan.go

@@ -43,6 +43,7 @@ type Component struct {
 	OtherComponents    []string
 	Owner              string
 	Project            string
+	SharedInfraVersion string
 	TerraformVersion   string
 	SiccMode           bool
 }
@@ -60,6 +61,7 @@ type Env struct {
 	Owner              string
 	Project            string
 	TerraformVersion   string
+	Type               string
 	SiccMode           bool
 
 	Components map[string]Component
@@ -170,6 +172,7 @@ func Print(p *Plan) error {
 			fmt.Printf("\t\t\t\tother_components: %v\n", component.OtherComponents)
 			fmt.Printf("\t\t\t\towner: %v\n", component.Owner)
 			fmt.Printf("\t\t\t\tproject: %v\n", component.Project)
+			fmt.Printf("\t\t\t\tshared_infra_version: %v\n", component.SharedInfraVersion)
 			fmt.Printf("\t\t\t\tterraform_version: %v\n", component.TerraformVersion)
 		}
 
@@ -273,6 +276,11 @@ func buildEnvs(conf *config.Config, siccMode bool) map[string]Env {
 		envPlan.InfraBucket = resolveRequired(defaults.InfraBucket, envConf.InfraBucket)
 		envPlan.Owner = resolveRequired(defaults.Owner, envConf.Owner)
 		envPlan.Project = resolveRequired(defaults.Project, envConf.Project)
+		if envConf.Type != nil {
+			envPlan.Type = *envConf.Type
+		} else {
+			envPlan.Type = "aws"
+		}
 		envPlan.SiccMode = siccMode
 
 		for componentName, componentConf := range conf.Envs[envName].Components {
@@ -290,6 +298,7 @@ func buildEnvs(conf *config.Config, siccMode bool) map[string]Env {
 			componentPlan.InfraBucket = resolveRequired(envPlan.InfraBucket, componentConf.InfraBucket)
 			componentPlan.Owner = resolveRequired(envPlan.Owner, componentConf.Owner)
 			componentPlan.Project = resolveRequired(envPlan.Project, componentConf.Project)
+			componentPlan.SharedInfraVersion = resolveRequired(conf.Defaults.SharedInfraVersion, componentConf.SharedInfraVersion)
 
 			componentPlan.Env = envName
 			componentPlan.Component = componentName

templates/module-invocation/main.tf.tmpl

@@ -0,0 +1,6 @@
+module "{{.ModuleName}}" {
+  source = "{{.ModuleSource}}"
+  {{range .Variables -}}
+    {{.}} = "${var.{{.}}}"
+  {{ end}}
+}

templates/module-invocation/outputs.tf.tmpl

@@ -0,0 +1,7 @@
+{{ $outer := . -}}
+{{- range .Outputs -}}
+output "{{.}}" {
+  value = "${module.{{$outer.ModuleName}}.{{.}}}"
+}
+
+{{end}}

templates/templates.go

@@ -3,19 +3,21 @@ package templates
 import "github.com/gobuffalo/packr"
 
 type T struct {
-	Account   packr.Box
-	Component packr.Box
-	Env       packr.Box
-	Global    packr.Box
-	Module    packr.Box
-	Repo      packr.Box
+	Account          packr.Box
+	Component        packr.Box
+	Env              packr.Box
+	Global           packr.Box
+	Module           packr.Box
+	ModuleInvocation packr.Box
+	Repo             packr.Box
 }
 
 var Templates = &T{
-	Account:   packr.NewBox("account"),
-	Component: packr.NewBox("component"),
-	Env:       packr.NewBox("env"),
-	Global:    packr.NewBox("global"),
-	Module:    packr.NewBox("module"),
-	Repo:      packr.NewBox("repo"),
+	Account:          packr.NewBox("account"),
+	Component:        packr.NewBox("component"),
+	Env:              packr.NewBox("env"),
+	Global:           packr.NewBox("global"),
+	Module:           packr.NewBox("module"),
+	ModuleInvocation: packr.NewBox("module-invocation"),
+	Repo:             packr.NewBox("repo"),
 }