speed up module bootstrapping … (#26)

* update vendor

* speed up module bootstrapping

Before we were doing a fetch on the module every time. On fast broadband this
is like 3-4 sec. But on tethering it is 100sec+.

This diff adds caching for modules and cached speed is negligable.

I couldn't directly use the terraform because it isn't well encapsulated, so
this diff means we don't support terraform registry modules and we only invaliate
the cache on new versions of fogg. This is ok because we only support the 1
hardcoded module for now.

* improve error handling

Gopkg.toml

@@ -38,6 +38,18 @@
   name = "github.com/hashicorp/terraform"
   version = "0.11.7"
 
+[[constraint]]
+  branch = "master"
+  name = "github.com/mitchellh/go-homedir"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/hashicorp/go-getter"
+
+[[constraint]]
+  name = "github.com/blang/semver"
+  version = "3.5.1"
+
 [[constraint]]
   name = "gopkg.in/src-d/go-git.v4"
   version = "4.5.0"

apply/apply.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -17,10 +16,6 @@ import (
 	"github.com/chanzuckerberg/fogg/util"
 	"github.com/gobuffalo/packr"
 	"github.com/hashicorp/hcl/hcl/printer"
-	tfConfig "github.com/hashicorp/terraform/config"
-	"github.com/hashicorp/terraform/config/module"
-	"github.com/hashicorp/terraform/svchost/auth"
-	"github.com/hashicorp/terraform/svchost/disco"
 	"github.com/pkg/errors"
 	"github.com/spf13/afero"
 )
@@ -244,25 +239,6 @@ func applyTemplate(sourceFile io.Reader, dest afero.Fs, path string, overrides i
 	return t.Execute(writer, overrides)
 }
 
-func downloadModule(dir, mod string) error {
-	disco := disco.NewDisco()
-	s := module.NewStorage(dir, disco, auth.NoCredentials)
-
-	return s.GetModule(dir, mod)
-}
-
-func downloadAndParseModule(mod string) (*tfConfig.Config, error) {
-	dir, e := ioutil.TempDir("", "fogg")
-	if e != nil {
-		return nil, errors.Wrap(e, "unable to create tempdir")
-	}
-	e = downloadModule(dir, mod)
-	if e != nil {
-		return nil, errors.Wrap(e, "unable to download module")
-	}
-	return tfConfig.LoadDir(dir)
-}
-
 // This should really be part of the plan stage, not apply. But going to
 // leave it here for now and re-think it when we make this mechanism
 // general purpose.
@@ -279,7 +255,7 @@ func applyModule(fs afero.Fs, path, mod string, box packr.Box) error {
 		return errors.Wrapf(e, "couldn't create %s directory", path)
 	}
 
-	c, e := downloadAndParseModule(mod)
+	c, e := util.DownloadAndParseModule(mod)
 	if e != nil {
 		return errors.Wrap(e, "could not download or parse module")
 	}

apply/apply_test.go

@@ -149,27 +149,10 @@ func TestCreateFile(t *testing.T) {
 // 	assert.Nil(t, e)
 // }
 
-func TestDownloadModule(t *testing.T) {
-	dir, e := ioutil.TempDir("", "fogg")
-	assert.Nil(t, e)
-
-	e = downloadModule(dir, "./test-module")
-	assert.Nil(t, e)
-	// TODO more asserts
-}
-
-func TestDownloadAndParseModule(t *testing.T) {
-	c, e := downloadAndParseModule("./test-module")
-	assert.Nil(t, e)
-	assert.NotNil(t, c)
-	assert.Len(t, c.Variables, 2)
-	assert.Len(t, c.Outputs, 2)
-}
-
 func TestApplyModule(t *testing.T) {
 	fs := afero.NewMemMapFs()
 
-	e := applyModule(fs, "mymodule", "./test-module", templates.Templates.ModuleInvocation)
+	e := applyModule(fs, "mymodule", "../util/test-module", templates.Templates.ModuleInvocation)
 	assert.Nil(t, e)
 
 	s, e := fs.Stat("mymodule")
@@ -181,7 +164,7 @@ func TestApplyModule(t *testing.T) {
 	r, e := afero.ReadFile(fs, "mymodule/main.tf")
 	assert.Nil(t, e)
 	expected := `module "test-module" {
-  source = "./test-module"
+  source = "../util/test-module"
   bar    = "${var.bar}"
   foo    = "${var.foo}"
 }

util/module_storage.go

@@ -0,0 +1,62 @@
+package util
+
+import (
+	"crypto/sha256"
+	"os"
+	"path/filepath"
+
+	getter "github.com/hashicorp/go-getter"
+	"github.com/hashicorp/terraform/config"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+)
+
+func DownloadModule(cacheDir, source string) (string, error) {
+	pwd, e := os.Getwd()
+	if e != nil {
+		return "", errors.Wrap(e, "could not get pwd")
+	}
+	s, e := getter.Detect(source, pwd, getter.Detectors)
+	if e != nil {
+		return "", errors.Wrap(e, "could not detect module type")
+	}
+
+	storage := &getter.FolderStorage{
+		StorageDir: cacheDir,
+	}
+	h := sha256.New()
+	_, e = h.Write([]byte(VersionCacheKey()))
+	if e != nil {
+		return "", errors.Wrap(e, "could not hash")
+	}
+	_, e = h.Write([]byte(source))
+	if e != nil {
+		return "", errors.Wrap(e, "could not hash")
+	}
+	hash := string(h.Sum(nil))
+
+	e = storage.Get(hash, s, false)
+	if e != nil {
+		return "", e
+	}
+	d, _, e := storage.Dir(hash)
+	if e != nil {
+		return "", errors.Wrap(e, "could not get module storage dir")
+	}
+	return d, nil
+}
+
+func DownloadAndParseModule(mod string) (*config.Config, error) {
+	homedir, e := homedir.Dir()
+	if e != nil {
+		return nil, errors.Wrap(e, "unable to find homedir")
+	}
+
+	dir := filepath.Join(homedir, ".fogg", "cache")
+
+	d, e := DownloadModule(dir, mod)
+	if e != nil {
+		return nil, errors.Wrap(e, "unable to download module")
+	}
+	return config.LoadDir(d)
+}

util/module_storage_test.go

@@ -0,0 +1,29 @@
+package util
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDownloadModule(t *testing.T) {
+	dir, e := ioutil.TempDir("", "fogg")
+	assert.Nil(t, e)
+
+	d, e := DownloadModule(dir, "./test-module")
+	assert.Nil(t, e)
+	assert.NotNil(t, d)
+	assert.NotEmpty(t, d)
+	// TODO more asserts
+}
+
+func TestDownloadAndParseModule(t *testing.T) {
+	c, e := DownloadAndParseModule("./test-module")
+	assert.Nil(t, e)
+	assert.NotNil(t, c)
+	assert.NotNil(t, c.Variables)
+	assert.NotNil(t, c.Outputs)
+	assert.Len(t, c.Variables, 2)
+	assert.Len(t, c.Outputs, 2)
+}

util/version.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strconv"
 
+	"github.com/blang/semver"
 	"github.com/pkg/errors"
 )
 
@@ -26,6 +27,15 @@ func VersionString() (string, error) {
 	return versionString(Version, GitSha, release, dirty), nil
 }
 
+func VersionCacheKey() string {
+	versionString, _ := VersionString()
+	v, e := semver.Parse(versionString)
+	if e != nil {
+		return ""
+	}
+	return fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
+}
+
 func versionString(version, sha string, release, dirty bool) string {
 	if release {
 		return version