template arch and os into plugin urls

errs/errs.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/go-errors/errors"
-	log "github.com/sirupsen/logrus"
 )
 
 type User struct {
@@ -49,7 +48,6 @@ func WrapUser(e error, msg string) error {
 }
 
 func WrapUserf(e error, msg string, a ...interface{}) error {
-	log.Debugf("wrapuser e: %#v", e)
 	if e == nil {
 		return nil
 	}

plugins/constants.go

@@ -3,12 +3,12 @@ package plugins
 const (
 	// TerraformPluginCacheDir is the directory where terraform caches tf approved providers
 	// See https://www.terraform.io/docs/configuration/providers.html#provider-plugin-cache
-	TerraformPluginCacheDir = ".terraform.d/plugin-cache"
+	// TerraformPluginCacheDir = ".terraform.d/plugin-cache"
+
 	// TerraformCustomPluginCacheDir is the directory used by terraform to search for custom providers
-	// We default to linux_amd64 since we're running terraform inside of docker
 	// We vendor providers here
 	// See https://www.terraform.io/docs/configuration/providers.html#third-party-plugins
-	TerraformCustomPluginCacheDir = "terraform.d/plugins/linux_amd64"
+	TerraformCustomPluginCacheDir = "terraform.d/plugins/{{.OS}}_{{.Arch}}"
 	// CustomPluginDir where we place custom binaries
 	CustomPluginDir = ".fogg/bin"
 )

plugins/custom_plugin.go

@@ -3,6 +3,7 @@ package plugins
 import (
 	"archive/tar"
 	"archive/zip"
+	"bytes"
 	"compress/gzip"
 	"fmt"
 	"io"
@@ -13,6 +14,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
+	"text/template"
 
 	"github.com/chanzuckerberg/fogg/errs"
 	log "github.com/sirupsen/logrus"
@@ -40,30 +42,55 @@ type CustomPlugin struct {
 
 // Install installs the custom plugin
 func (cp *CustomPlugin) Install(fs afero.Fs, pluginName string) error {
-	return cp.install(fs, pluginName, cp.URL, runtime.GOARCH, runtime.GOOS)
+	return cp.install(fs, pluginName, cp.URL, runtime.GOOS, runtime.GOARCH)
 }
 
 // Install delegates to install
 func (cp *CustomPlugin) install(
 	fs afero.Fs,
 	pluginName string,
 	url string,
+	pluginOS string,
 	pluginArch string,
-	pluginOS string) error {
+) error {
 	if cp == nil {
 		return errs.NewUser("nil CustomPlugin")
 	}
 	if fs == nil {
 		return errs.NewUser("nil fs")
 	}
-	tmpPath, err := cp.fetch(pluginName, url)
+
+	fullUrl, err := Template(url, pluginOS, pluginArch)
+	if err != nil {
+		return err
+	}
+	tmpPath, err := cp.fetch(pluginName, fullUrl)
 	defer os.Remove(tmpPath)
 	if err != nil {
 		return err
 	}
 	return cp.process(fs, pluginName, tmpPath, cp.targetDir)
 }
 
+func Template(url, os, arch string) (string, error) {
+	data := struct {
+		OS   string
+		Arch string
+	}{os, arch}
+
+	t, err := template.New("url").Parse(url)
+	if err != nil {
+		return "", errs.WrapUserf(err, "could not parse url template %s", url)
+	}
+	buf := bytes.NewBuffer(nil)
+	err = t.Execute(buf, data)
+	if err != nil {
+		return "", err
+	}
+	out := buf.String()
+	return out, nil
+}
+
 // SetTargetPath sets the target path for this plugin
 func (cp *CustomPlugin) SetTargetPath(path string) {
 	cp.targetDir = path
@@ -75,6 +102,7 @@ func (cp *CustomPlugin) fetch(pluginName string, url string) (string, error) {
 	if err != nil {
 		return "", errs.WrapUser(err, "could not create temporary directory") //FIXME
 	}
+	log.Debugf("downloading %s to tempfile", url)
 	resp, err := http.Get(url)
 	if err != nil {
 		return "", errs.WrapUserf(err, "could not get %s", url) // FIXME
@@ -99,29 +127,38 @@ func (cp *CustomPlugin) process(fs afero.Fs, pluginName string, path string, tar
 }
 
 func (cp *CustomPlugin) processBin(fs afero.Fs, name string, downloadPath string, targetDir string) error {
-	err := fs.MkdirAll(targetDir, 0755)
+	target, _ := Template(targetDir, runtime.GOOS, runtime.GOARCH)
+
+	err := fs.MkdirAll(target, 0755)
 	if err != nil {
-		return errs.WrapUserf(err, "Could not create directory %s", targetDir)
+		return errs.WrapUserf(err, "Could not create directory %s", target)
 	}
-	targetPath := path.Join(targetDir, name)
+
+	targetPath := path.Join(target, name)
 	src, err := os.Open(downloadPath)
 	if err != nil {
 		return errs.WrapUserf(err, "Could not open downloaded file at %s", downloadPath)
 	}
+
 	dst, err := fs.Create(targetPath)
 	if err != nil {
 		return errs.WrapUserf(err, "Could not open target file at %s", targetPath)
 	}
+
 	_, err = io.Copy(dst, src)
 	if err != nil {
 		return errs.WrapUserf(err, "Could not move %s to %s", downloadPath, targetPath)
 	}
+
 	err = fs.Chmod(targetPath, os.FileMode(0755))
 	return errs.WrapUserf(err, "Error making %s executable", targetPath)
 }
 
 // https://medium.com/@skdomino/taring-untaring-files-in-go-6b07cf56bc07
 func (cp *CustomPlugin) processTar(fs afero.Fs, path string, targetDir string) error {
+	targetDir, _ = Template(targetDir, runtime.GOOS, runtime.GOARCH)
+	log.Debugf("untarring from %s to %s", path, targetDir)
+
 	err := fs.MkdirAll(targetDir, 0755)
 	if err != nil {
 		return errs.WrapUserf(err, "Could not create directory %s", targetDir)
@@ -177,6 +214,8 @@ func (cp *CustomPlugin) processTar(fs afero.Fs, path string, targetDir string) e
 
 // based on https://golangcode.com/create-zip-files-in-go/
 func (cp *CustomPlugin) processZip(fs afero.Fs, downloadPath string, targetDir string) error {
+	targetDir, _ = Template(targetDir, runtime.GOOS, runtime.GOARCH)
+
 	err := fs.MkdirAll(targetDir, 0755)
 	if err != nil {
 		return errs.WrapUserf(err, "Could not create directory %s", targetDir)

plugins/custom_plugin_test.go

@@ -192,3 +192,35 @@ func generateZip(t *testing.T, files []string) string {
 	}
 	return f.Name()
 }
+
+func TestTemplate(t *testing.T) {
+	type args struct {
+		url  string
+		arch string
+		os   string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{"no-op", args{"foo", "bar", "bam"}, "foo", false},
+		{"os", args{"{{.OS}}", "bar", "bam"}, "bar", false},
+		{"arch", args{"{{.Arch}}", "bar", "bam"}, "bam", false},
+		{"os_arch", args{"{{.OS}}_{{.Arch}}", "bar", "bam"}, "bar_bam", false},
+		{"bad template", args{"{{.asdf", "bar", "bam"}, "", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := plugins.Template(tt.args.url, tt.args.arch, tt.args.os)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Template() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("Template() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

templates/repo/scripts/common.mk

@@ -8,26 +8,25 @@ AUTO_APPROVE := false
 # We need to do this because `terraform fmt` recurses into .terraform/modules
 # and wont' accept more than one file at a time.
 TF=$(wildcard *.tf)
-IMAGE_VERSION=$(DOCKER_IMAGE_VERSION)_TF$(TERRAFORM_VERSION)
-ifndef USE_DOCKER
-TFENV_DIR ?= $(HOME)/.tfenv
-export PATH :=$(TFENV_DIR)/versions/$(TERRAFORM_VERSION)/:$(PATH)
-endif
 
-docker_base = \
-	docker run --rm -e HOME=/home -v $$HOME/.aws:/home/.aws -v $(REPO_ROOT):/repo \
-	-v $(REPO_ROOT)/.fogg/bin:/usr/local/bin -v $(REPO_ROOT)/terraform.d:/repo/$(REPO_RELATIVE_PATH)/terraform.d \
-	-e GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' \
-	-e RUN_USER_ID=$(shell id -u) -e RUN_GROUP_ID=$(shell id -g) \
-	-e TF_PLUGIN_CACHE_DIR="/repo/.terraform.d/plugin-cache" -e TF="$(TF)" \
-	-w /repo/$(REPO_RELATIVE_PATH) $(TF_VARS) $(FOGG_DOCKER_FLAGS) $$(sh $(REPO_ROOT)/scripts/docker-ssh-mount.sh)
-docker_terraform = $(docker_base) chanzuckerberg/terraform:$(IMAGE_VERSION)
-docker_sh = $(docker_base) --entrypoint='/bin/sh' chanzuckerberg/terraform:$(IMAGE_VERSION)
 
 ifdef USE_DOCKER
+	IMAGE_VERSION=$(DOCKER_IMAGE_VERSION)_TF$(TERRAFORM_VERSION)
+	docker_base = \
+		docker run --rm -e HOME=/home -v $$HOME/.aws:/home/.aws -v $(REPO_ROOT):/repo \
+		-v $(REPO_ROOT)/.fogg/bin:/usr/local/bin -v $(REPO_ROOT)/terraform.d:/repo/$(REPO_RELATIVE_PATH)/terraform.d \
+		-e GIT_SSH_COMMAND='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' \
+		-e RUN_USER_ID=$(shell id -u) -e RUN_GROUP_ID=$(shell id -g) \
+		-e TF_PLUGIN_CACHE_DIR="/repo/.terraform.d/plugin-cache" -e TF="$(TF)" \
+		-w /repo/$(REPO_RELATIVE_PATH) $(TF_VARS) $(FOGG_DOCKER_FLAGS) $$(sh $(REPO_ROOT)/scripts/docker-ssh-mount.sh)
+	docker_terraform = $(docker_base) chanzuckerberg/terraform:$(IMAGE_VERSION)
+	docker_sh = $(docker_base) --entrypoint='/bin/sh' chanzuckerberg/terraform:$(IMAGE_VERSION)
 	sh_command ?= $(docker_sh)
 	terraform_command ?= $(docker_terraform)
 else
+	TFENV_DIR ?= $(HOME)/.tfenv
+	export PATH :=$(TFENV_DIR)/versions/$(TERRAFORM_VERSION)/:$(PATH)
+	export TF_PLUGIN_CACHE_DIR=$(REPO_ROOT)/.terraform.d/plugin-cache
 	sh_command ?= $(SHELL)
 	terraform_command ?= $(TFENV_DIR)/versions/$(TERRAFORM_VERSION)/terraform
 endif