add remote support (#16)

Co-authored-by: nxcc <>

README.md

@@ -32,22 +32,21 @@ in this repository (in that order) are good starting points.*
   * Load whole directories as key/values data at once, e.g. into a ConfigMap
   * Load structured data (JSON/YAML/env) into a CUE struct
   * Automatically decrypt [SOPS][SOPS]-encrypted data
+  * Load remote charts
 
 ## Install
 Download the [latest release][rel] or build with *go1.20rc3 or later*:
 
     go install github.com/noris-network/cuegen@latest
 
-To use cuegen as kustomize plugin, find instructions in the [kustomize example][kusteg].
-
 
 ## Usage
-Cuegen can be used stand-alone or as generator in [kustomize][kust]
-(see [example](examples/kustomize/)).
 
     cuegen path/to/cuegen.yaml
     # or
     cuegen path/to/directory-containing-cuegen-dot-yaml
+    # or
+    cuegen https://git.example.com/deployments/myapp.git
 
 Have a look at the [examples][eg] for some ready-to-run examples.
 
@@ -58,8 +57,8 @@ expects the cuegen config file to be named `cuegen.yaml`.
 
 ## Configuration
 A configuration file (preferred name: `cuegen.cue`, [schema][cfgschema]) is
-required to run `cuegen`. For backwards compatibility, and for `cuegen` to
-work as a kustomize plugin, the yaml format will still be supported in the future.
+required to run `cuegen`. For backwards compatibility the yaml format will still
+be supported in the future.
 
     cuegen: {
       objectsPath:    "objects"          // this will be dumped to YAML
@@ -78,6 +77,15 @@ work as a kustomize plugin, the yaml format will still be supported in the futur
       dumpOverlays: false                // dump overlays for debugging
     }
 
+## Environment Variables
+Some environment variables can help working with cuegen:
+
+    CUEGEN_HTTP_USERNAME      username for git authentication
+    CUEGEN_HTTP_PASSWORD      password for git authentication
+    SOPS_AGE_KEY              age key for decryption of files
+    SOPS_AGE_KEY_FILE         age key file for decryption of files
+    CUEGEN_DEBUG              turn on debug output with "true"
+
 ## Components
 Components can be
 
@@ -187,14 +195,14 @@ Load all files from directory `scripts` as key/values into `configMap.scripts.da
   * `v0.6.0` - add dumpOverlays option
   * `v0.7.0` - upgrade cue to v0.5.0 (many fixes, rare performance regression still present)
   * `v0.7.1` - fix secret handling of @readfile
+  * `v0.7.2` - internal cleanup
+  * `v0.8.0` - allow remote cuegen directories, rm kustomize plugin support
 
 [CUE]:         https://cuelang.org
 [SOPS]:        https://github.com/mozilla/sops
-[kust]:        https://kustomize.io/
 [k8stut]:      https://cuelang.org/docs/tutorials/
 [eg]:          examples/
 [rel]:         https://github.com/noris-network/cuegen/releases/latest
-[kusteg]:      examples/kustomize
 [cmp]:         https://argo-cd.readthedocs.io/en/stable/user-guide/config-management-plugins/#option-2-configure-plugin-via-sidecar
 [cuegen-cmp]:  https://hub.docker.com/r/nxcc/cuegen-cmp
 [expenv]:      https://pkg.go.dev/os#ExpandEnv

examples/readme.md

@@ -3,10 +3,10 @@
 * [configmap](configmap): load external data into a ConfigMap
 * [values](values): load external structured data into a CUE value
 * [encrypted](encrypted): load and decrypt [SOPS][SOPS]-encrypted data
-* [kustomize](kustomize): use `cuegen` as kustomize plugin
 * [components](components): compose several charts into one
 * [control-repository](control-repository): full example how to install several
   instances of an application in various environments
+* [remote](remote): render chart from remote repository
 
 To run these examples, put `cuegen` into your `PATH` and execute
 

examples/remote/readme.md

@@ -0,0 +1,11 @@
+## Load Remote charts
+
+As no local data is required, just run
+
+    cuegen https://github.com/nxcc/cuegen-remote-test.git
+
+to render the chart located in the given repository. Environment Variables in the
+given URL are expanded. Like in components, `ref` and `#`-prefixed subpaths can be used:
+
+    cuegen "https://github.com/nxcc/cuegen-remote-test.git?ref=subpath#apps/app_b"
+

go.mod

@@ -4,10 +4,10 @@ go 1.20
 
 require (
 	cuelang.org/go v0.5.0
-	github.com/cue-exp/cueconfig v0.0.1
 	github.com/forensicanalysis/gitfs v0.2.1
 	github.com/go-git/go-git/v5 v5.7.0
 	github.com/joho/godotenv v1.5.1
+	github.com/nxcc/cueconfig v0.0.1
 	github.com/rogpeppe/go-internal v1.10.0
 	go.mozilla.org/sops/v3 v3.7.3
 	golang.org/x/exp v0.0.0-20230202163644-54bba9f4231b

go.sum

@@ -84,8 +84,6 @@ github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b80
 github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw=
 github.com/containerd/continuity v0.2.2 h1:QSqfxcn8c+12slxwu00AtzXrsami0MJb/MQs9lOLHLA=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/cue-exp/cueconfig v0.0.1 h1:p8CZgrs3Ion58Agwyl2rYrguy/zNDS7vnBPL1TZOL8k=
-github.com/cue-exp/cueconfig v0.0.1/go.mod h1:NmMZe8inhcWk4gc0iCpDBQd9slapQawwApS3n2XF/P4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -281,6 +279,8 @@ github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9
 github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxcc/cueconfig v0.0.1 h1:rnuTlLODSnZNrH84TJW1DldeaAA/jTLZiRg7WryhQR0=
+github.com/nxcc/cueconfig v0.0.1/go.mod h1:KWh7yngeWWLCR2GShDV2P47m+25fjBVEgfGsZYZJ9AI=
 github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
 github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=

internal/app/main.go

@@ -20,21 +20,22 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/fs"
 	"log"
 	"os"
 	"path/filepath"
 	"runtime/debug"
+	"strings"
 
-	"github.com/cue-exp/cueconfig"
 	"github.com/noris-network/cuegen/internal/cuegen"
+	"github.com/nxcc/cueconfig"
 	"gopkg.in/yaml.v3"
 )
 
 const defaultYamlCuegenFile = "cuegen.yaml"
 const defaultCueCuegenFile = "cuegen.cue"
 
 var Build = ""
-var runningAsKustomizePlugin = os.Getenv("KUSTOMIZE_PLUGIN_CONFIG_ROOT") != ""
 
 func Main() int {
 
@@ -92,16 +93,7 @@ func Main() int {
 	}
 
 	// setup paths
-	chartRoot := ""
-	if runningAsKustomizePlugin {
-		cwd, err := os.Getwd()
-		if err != nil {
-			log.Fatalf("getwd: %v", err)
-		}
-		chartRoot = cwd
-	} else {
-		chartRoot = filepath.Dir(configFile)
-	}
+	chartRoot := filepath.Dir(configFile)
 
 	chartRoot, err = filepath.Abs(chartRoot)
 	if err != nil {
@@ -123,8 +115,24 @@ func Main() int {
 // loadConfig loads the cuegen config. When a directory is passed, cuegen will
 // look for the default "cuegen.yaml" in that directory.
 func loadConfig(path string) (string, cuegen.Config, error) {
+
+	var rootfs fs.FS
+
+	if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+		path = os.ExpandEnv(path)
+		gitfs, err := cuegen.GetGitFS(path)
+		if err != nil {
+			return "", cuegen.Config{}, fmt.Errorf("in loadConfig: %v", err)
+		}
+		rootfs = gitfs
+		path = "."
+	} else {
+		rootfs = os.DirFS(".")
+		path = strings.TrimRight(path, "/")
+	}
+
 	file, err := func() (string, error) {
-		fileInfo, err := os.Stat(path)
+		fileInfo, err := fs.Stat(rootfs, path)
 		if err != nil {
 			return "", fmt.Errorf("stat: %v", err)
 		}
@@ -134,13 +142,13 @@ func loadConfig(path string) (string, cuegen.Config, error) {
 		if fileInfo.IsDir() {
 			// cuegen.cue?
 			file := filepath.Join(path, defaultCueCuegenFile)
-			fileInfo, err := os.Stat(file)
+			fileInfo, err := fs.Stat(rootfs, file)
 			if err == nil && fileInfo.Mode().IsRegular() {
 				return file, nil
 			}
 			// cuegen.yaml?
 			file = filepath.Join(path, defaultYamlCuegenFile)
-			fileInfo, err = os.Stat(file)
+			fileInfo, err = fs.Stat(rootfs, file)
 			if err == nil && fileInfo.Mode().IsRegular() {
 				return file, nil
 			}
@@ -155,17 +163,17 @@ func loadConfig(path string) (string, cuegen.Config, error) {
 	}
 	ext := filepath.Ext(file)
 	if ext == ".cue" {
-		return loadCueConfig(file)
+		return loadCueConfig(rootfs, file)
 	}
-	if ext == ".yml" || ext == ".yaml" || runningAsKustomizePlugin {
-		return loadYamlConfig(file)
+	if ext == ".yml" || ext == ".yaml" {
+		return loadYamlConfig(rootfs, file)
 	}
 	return "", cuegen.Config{}, errors.New("no config found")
 }
 
 // loadYamlConfig loads the cuegen config
-func loadYamlConfig(file string) (string, cuegen.Config, error) {
-	fh, err := os.Open(file)
+func loadYamlConfig(fsys fs.FS, file string) (string, cuegen.Config, error) {
+	fh, err := fsys.Open(file)
 	if err != nil {
 		return "", cuegen.Config{}, err
 	}
@@ -174,21 +182,23 @@ func loadYamlConfig(file string) (string, cuegen.Config, error) {
 	decoder.KnownFields(true)
 	if err := decoder.Decode(&config); err != nil {
 		if errors.Is(err, io.EOF) {
-			return file, cuegen.Config{}, nil
+			return file, cuegen.Config{RootFS: &fsys}, nil
 		}
 		return "", cuegen.Config{}, err
 	}
+	config.RootFS = &fsys
 	return file, config, nil
 }
 
 //go:embed schema.cue
 var cuegenConfigSchema []byte
 
 // loadCueConfig loads the cuegen config
-func loadCueConfig(file string) (string, cuegen.Config, error) {
+func loadCueConfig(fsys fs.FS, file string) (string, cuegen.Config, error) {
 	config := struct{ Cuegen cuegen.Config }{}
-	if err := cueconfig.Load(file, cuegenConfigSchema, nil, nil, &config); err != nil {
+	if err := cueconfig.LoadFS(fsys, file, cuegenConfigSchema, nil, nil, &config); err != nil {
 		return "", cuegen.Config{}, fmt.Errorf("load cue: %v", err)
 	}
+	config.Cuegen.RootFS = &fsys
 	return file, config.Cuegen, nil
 }

internal/cuegen/main.go

@@ -46,6 +46,7 @@ type Config struct {
 	ChartRoot      string
 	CheckPath      string   `yaml:"checkPath"`
 	CheckPaths     []string `yaml:"checkPaths"`
+	RootFS         *fs.FS
 }
 
 type Component struct {
@@ -65,11 +66,16 @@ type Cuegen struct {
 	CheckPaths     []string
 	ChartRoot      string
 	DumpOverlays   bool
+	RootFS         *fs.FS
 }
 
 // Exec initializes the Cuegen struct and executes cuegen
 func Exec(config Config) error {
 
+	if os.Getenv("CUEGEN_DEBUG") == "true" {
+		config.Debug = true
+	}
+
 	// apply preferred defaults when all required fields are empty
 	if config.ObjectsPath == "" &&
 		config.SecretDataPath == "" &&
@@ -88,6 +94,7 @@ func Exec(config Config) error {
 		ChartRoot:      config.ChartRoot,
 		SecretDataPath: config.SecretDataPath,
 		DumpOverlays:   config.DumpOverlays,
+		RootFS:         config.RootFS,
 	}
 
 	if config.CheckPath != "" {
@@ -105,6 +112,13 @@ func Exec(config Config) error {
 	}
 	cg.Components = components
 
+	if cg.RootFS != nil {
+		cg.Components["remoteRootFS"] = Component{
+			Filesystem: *cg.RootFS,
+			Type:       "remoterootfs",
+		}
+	}
+
 	if cg.Debug {
 		cg.PrintConfig()
 	}
@@ -115,12 +129,13 @@ func Exec(config Config) error {
 // getComponents loads components from the given paths
 func (cg Cuegen) getComponents(componentPaths []string) (Components, error) {
 	components := Components{}
+
 	for _, componentPath := range componentPaths {
 		component := Component{Path: componentPath, ID: generateID(componentPath)}
 		switch {
 
 		case strings.HasPrefix(componentPath, "http://") || strings.HasPrefix(componentPath, "https://") || strings.HasPrefix(componentPath, "git@"):
-			gitfs, err := getGitFS(componentPath)
+			gitfs, err := GetGitFS(componentPath)
 			if err != nil {
 				return nil, fmt.Errorf("getComponents: %v", err)
 			}
@@ -158,11 +173,11 @@ func generateID(name string) string {
 	return base32.StdEncoding.EncodeToString(bs[:])[:10]
 }
 
-// getGitFS returns a fs.FS from the given git repository URL
-func getGitFS(component string) (fs.FS, error) {
+// GetGitFS returns a fs.FS from the given git repository URL
+func GetGitFS(component string) (fs.FS, error) {
 	uri, ref, subDir, err := parseGitURL(component)
 	if err != nil {
-		return nil, fmt.Errorf("getGitURL: open: %v", err)
+		return nil, fmt.Errorf("GetGitURL: open: %v", err)
 	}
 
 	opts := git.CloneOptions{