Strict Environment Handling

cli/integration_tests/strict_env_vars/global_passthrough_env.t

@@ -0,0 +1,69 @@
+Setup
+  $ . ${TESTDIR}/../setup.sh
+  $ . ${TESTDIR}/setup.sh $(pwd) global_passthrough_env
+
+Strict + []
+  $ ${TURBO} build --env=strict -vv --filter=nothing > /dev/null
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Global turbo version: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Repository Root: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Running command as global turbo (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::cli: pkg_inference_root set to "" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re)
+  [-0-9:.TWZ+]+ \[INFO]  turbo: skipping turbod since we appear to be in a non-interactive context (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: new global hash: value=ef18881222fd6a72 (re)
+  [-0-9:.TWZ+]+ |[DEBUG] turbo: local cache folder: path="" (re)
+  No tasks were executed as part of this run.
+
+Infer + []
+  $ ${TURBO} build --env=infer -vv --filter=nothing > /dev/null
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Global turbo version: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Repository Root: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Running command as global turbo (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::cli: pkg_inference_root set to "" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re)
+  [-0-9:.TWZ+]+ \[INFO]  turbo: skipping turbod since we appear to be in a non-interactive context (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: new global hash: value=9cda29cc8ed880a1 (re)
+  [-0-9:.TWZ+]+ |[DEBUG] turbo: local cache folder: path="" (re)
+  No tasks were executed as part of this run.
+
+Loose + []
+  $ ${TURBO} build --env=loose -vv --filter=nothing > /dev/null
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Global turbo version: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Repository Root: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Running command as global turbo (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::cli: pkg_inference_root set to "" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re)
+  [-0-9:.TWZ+]+ \[INFO]  turbo: skipping turbod since we appear to be in a non-interactive context (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: new global hash: value=b7e2e8ed3383c982 (re)
+  [-0-9:.TWZ+]+ |[DEBUG] turbo: local cache folder: path="" (re)
+  No tasks were executed as part of this run.
+
+Loose + ["BONUS_VAR"]
+  $ rm turbo.json
+  $ mv turbo_loose.json turbo.json
+  $ ${TURBO} build --env=loose -vv --filter=nothing > /dev/null
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Global turbo version: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Repository Root: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Running command as global turbo (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::cli: pkg_inference_root set to "" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re)
+  [-0-9:.TWZ+]+ \[INFO]  turbo: skipping turbod since we appear to be in a non-interactive context (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: new global hash: value=b7e2e8ed3383c982 (re)
+  [-0-9:.TWZ+]+ |[DEBUG] turbo: local cache folder: path="" (re)
+  No tasks were executed as part of this run.

cli/integration_tests/strict_env_vars/global_passthrough_env/.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+.turbo
+.npmrc

cli/integration_tests/strict_env_vars/global_passthrough_env/apps/my-app/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "my-app",
+  "scripts": {
+    "build": "echo 'building'"
+  },
+  "dependencies": {
+    "util": "*"
+  }
+}

cli/integration_tests/strict_env_vars/global_passthrough_env/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "monorepo",
+  "workspaces": [
+    "apps/**",
+    "packages/**"
+  ]
+}

cli/integration_tests/strict_env_vars/global_passthrough_env/packages/util/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "util",
+  "scripts": {
+    "build": "echo 'building'"
+  }
+}

cli/integration_tests/strict_env_vars/global_passthrough_env/turbo.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": ["dist/**"]
+    }
+  },
+  "globalPassthroughEnv": []
+}

cli/integration_tests/strict_env_vars/global_passthrough_env/turbo_loose.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": ["dist/**"]
+    }
+  },
+  "globalPassthroughEnv": ["BONUS_VAR"]
+}

cli/integration_tests/strict_env_vars/monorepo/.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+.turbo
+.npmrc

cli/integration_tests/strict_env_vars/monorepo/apps/my-app/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "my-app",
+  "scripts": {
+    "build": "echo 'building'"
+  },
+  "dependencies": {
+    "util": "*"
+  }
+}

cli/integration_tests/strict_env_vars/monorepo/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "monorepo",
+  "workspaces": [
+    "apps/**",
+    "packages/**"
+  ]
+}

cli/integration_tests/strict_env_vars/monorepo/packages/util/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "util",
+  "scripts": {
+    "build": "echo 'building'"
+  }
+}

cli/integration_tests/strict_env_vars/monorepo/turbo.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://turbo.build/schema.json",
+  "pipeline": {
+    "build": {
+      "outputs": ["dist/**"]
+    }
+  }
+}

cli/integration_tests/strict_env_vars/setup.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+TARGET_DIR=$1
+FIXTURE=$2
+
+cp -a ${SCRIPT_DIR}/$2/. ${TARGET_DIR}/
+${SCRIPT_DIR}/../setup_git.sh ${TARGET_DIR}

cli/integration_tests/strict_env_vars/strict_env_vars.t

@@ -0,0 +1,67 @@
+Setup
+  $ . ${TESTDIR}/../setup.sh
+  $ . ${TESTDIR}/setup.sh $(pwd) monorepo
+
+Strict with no passthroughEnv opts you into the new hashing behavior.
+  $ ${TURBO} build --env=strict -vv --filter=nothing > /dev/null
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Global turbo version: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Repository Root: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Running command as global turbo (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::cli: pkg_inference_root set to "" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re)
+  [-0-9:.TWZ+]+ \[INFO]  turbo: skipping turbod since we appear to be in a non-interactive context (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: new global hash: value=ef18881222fd6a72 (re)
+  [-0-9:.TWZ+]+ |[DEBUG] turbo: local cache folder: path="" (re)
+  No tasks were executed as part of this run.
+
+Loose with no passthroughEnv also opts you into the new hashing behavior.
+  $ ${TURBO} build --env=loose -vv --filter=nothing > /dev/null
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Global turbo version: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Repository Root: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Running command as global turbo (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::cli: pkg_inference_root set to "" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re)
+  [-0-9:.TWZ+]+ \[INFO]  turbo: skipping turbod since we appear to be in a non-interactive context (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: new global hash: value=b7e2e8ed3383c982 (re)
+  [-0-9:.TWZ+]+ |[DEBUG] turbo: local cache folder: path="" (re)
+  No tasks were executed as part of this run.
+
+Infer with no passthroughEnv keeps you on the old hashing
+  $ ${TURBO} build --env=infer -vv --filter=nothing > /dev/null
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Global turbo version: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Repository Root: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Running command as global turbo (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::cli: pkg_inference_root set to "" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re)
+  [-0-9:.TWZ+]+ \[INFO]  turbo: skipping turbod since we appear to be in a non-interactive context (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=f71f6eda617fc880 (re)
+  [-0-9:.TWZ+]+ |[DEBUG] turbo: local cache folder: path="" (re)
+  No tasks were executed as part of this run.
+
+Weird arg for specifying infer, same old hashing
+  $ ${TURBO} build --env -vv --filter=nothing > /dev/null
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Global turbo version: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: No local turbo binary found at: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Repository Root: .* (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::shim: Running command as global turbo (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turborepo_lib::cli: pkg_inference_root set to "" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re)
+  [-0-9:.TWZ+]+ \[INFO]  turbo: skipping turbod since we appear to be in a non-interactive context (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re)
+  [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=f71f6eda617fc880 (re)
+  [-0-9:.TWZ+]+ |[DEBUG] turbo: local cache folder: path="" (re)
+  No tasks were executed as part of this run.

cli/internal/env/env.go

@@ -35,6 +35,11 @@ func (evm EnvironmentVariableMap) Merge(another EnvironmentVariableMap) {
 	}
 }
 
+// Add creates one new environment variable.
+func (evm EnvironmentVariableMap) Add(key string, value string) {
+	evm[key] = value
+}
+
 // Names returns a sorted list of env var names for the EnvironmentVariableMap
 func (evm EnvironmentVariableMap) Names() []string {
 	names := []string{}
@@ -85,7 +90,8 @@ func (evm EnvironmentVariableMap) ToHashable() EnvironmentVariablePairs {
 	})
 }
 
-func getEnvMap() EnvironmentVariableMap {
+// GetEnvMap returns a map of env vars and their values from os.Environ
+func GetEnvMap() EnvironmentVariableMap {
 	envMap := make(map[string]string)
 	for _, envVar := range os.Environ() {
 		if i := strings.Index(envVar, "="); i >= 0 {
@@ -96,8 +102,8 @@ func getEnvMap() EnvironmentVariableMap {
 	return envMap
 }
 
-// fromKeys returns a map of env vars and their values from a given set of env var names
-func fromKeys(all EnvironmentVariableMap, keys []string) EnvironmentVariableMap {
+// FromKeys returns a map of env vars and their values from a given set of env var names
+func FromKeys(all EnvironmentVariableMap, keys []string) EnvironmentVariableMap {
 	output := EnvironmentVariableMap{}
 	for _, key := range keys {
 		output[key] = all[key]
@@ -138,14 +144,14 @@ func fromMatching(all EnvironmentVariableMap, keyMatchers []string, shouldExclud
 
 // GetHashableEnvVars returns all sorted key=value env var pairs for both frameworks and from envKeys
 func GetHashableEnvVars(keys []string, matchers []string, envVarContainingExcludePrefix string) (DetailedMap, error) {
-	all := getEnvMap()
+	all := GetEnvMap()
 
 	detailedMap := DetailedMap{
 		All:      EnvironmentVariableMap{},
 		BySource: BySource{},
 	}
 
-	detailedMap.BySource.Explicit = fromKeys(all, keys)
+	detailedMap.BySource.Explicit = FromKeys(all, keys)
 	detailedMap.All.Merge(detailedMap.BySource.Explicit)
 
 	// Create an excluder function to pass to matcher.

cli/internal/fs/testdata/correct/turbo.json

@@ -2,6 +2,7 @@
 {
   "pipeline": {
     "build": {
+      "passthroughEnv": ["GITHUB_TOKEN"],
       // mocked test comment
       "dependsOn": [
         // mocked test comment
@@ -40,6 +41,7 @@
   },
   "globalDependencies": ["some-file", "../another-dir/**", "$GLOBAL_ENV_VAR"],
   "globlaEnv": ["SOME_VAR", "ANOTHER_VAR"],
+  "globalPassthroughEnv": ["AWS_SECRET_KEY"],
   "remoteCache": {
     "teamId": "team_id",
     "signature": true