From 815a0c3765859e791335be5c5b65f27d41d87e12 Mon Sep 17 00:00:00 2001 From: nxcc <73293562+nxcc@users.noreply.github.com> Date: Tue, 4 Jul 2023 11:34:44 +0200 Subject: [PATCH] add remote support (#16) Co-authored-by: nxcc <> --- README.md | 24 ++++++++---- examples/kustomize/cuegen.yaml | 8 ---- examples/kustomize/greeting.txt | 1 - examples/kustomize/kustomization.yaml | 5 --- examples/kustomize/main.cue | 13 ------- examples/kustomize/readme.md | 17 -------- examples/readme.md | 2 +- examples/remote/readme.md | 11 ++++++ go.mod | 2 +- go.sum | 4 +- internal/app/main.go | 56 ++++++++++++++++----------- internal/cuegen/main.go | 23 +++++++++-- scripts/test-all-examples.sh | 11 ++---- tests/local/kustomize.txtar | 47 ---------------------- tests/remote/remote-root.txtar | 32 +++++++++++++++ 15 files changed, 118 insertions(+), 138 deletions(-) delete mode 100644 examples/kustomize/cuegen.yaml delete mode 100644 examples/kustomize/greeting.txt delete mode 100644 examples/kustomize/kustomization.yaml delete mode 100644 examples/kustomize/main.cue delete mode 100644 examples/kustomize/readme.md create mode 100644 examples/remote/readme.md delete mode 100644 tests/local/kustomize.txtar create mode 100644 tests/remote/remote-root.txtar diff --git a/README.md b/README.md index af980b4..996bbb1 100644 --- a/README.md +++ b/README.md @@ -32,22 +32,21 @@ in this repository (in that order) are good starting points.* * Load whole directories as key/values data at once, e.g. into a ConfigMap * Load structured data (JSON/YAML/env) into a CUE struct * Automatically decrypt [SOPS][SOPS]-encrypted data + * Load remote charts ## Install Download the [latest release][rel] or build with *go1.20rc3 or later*: go install github.com/noris-network/cuegen@latest -To use cuegen as kustomize plugin, find instructions in the [kustomize example][kusteg]. - ## Usage -Cuegen can be used stand-alone or as generator in [kustomize][kust] -(see [example](examples/kustomize/)). cuegen path/to/cuegen.yaml # or cuegen path/to/directory-containing-cuegen-dot-yaml + # or + cuegen https://git.example.com/deployments/myapp.git Have a look at the [examples][eg] for some ready-to-run examples. @@ -58,8 +57,8 @@ expects the cuegen config file to be named `cuegen.yaml`. ## Configuration A configuration file (preferred name: `cuegen.cue`, [schema][cfgschema]) is -required to run `cuegen`. For backwards compatibility, and for `cuegen` to -work as a kustomize plugin, the yaml format will still be supported in the future. +required to run `cuegen`. For backwards compatibility the yaml format will still +be supported in the future. cuegen: { objectsPath: "objects" // this will be dumped to YAML @@ -78,6 +77,15 @@ work as a kustomize plugin, the yaml format will still be supported in the futur dumpOverlays: false // dump overlays for debugging } +## Environment Variables +Some environment variables can help working with cuegen: + + CUEGEN_HTTP_USERNAME username for git authentication + CUEGEN_HTTP_PASSWORD password for git authentication + SOPS_AGE_KEY age key for decryption of files + SOPS_AGE_KEY_FILE age key file for decryption of files + CUEGEN_DEBUG turn on debug output with "true" + ## Components Components can be @@ -187,14 +195,14 @@ Load all files from directory `scripts` as key/values into `configMap.scripts.da * `v0.6.0` - add dumpOverlays option * `v0.7.0` - upgrade cue to v0.5.0 (many fixes, rare performance regression still present) * `v0.7.1` - fix secret handling of @readfile + * `v0.7.2` - internal cleanup + * `v0.8.0` - allow remote cuegen directories, rm kustomize plugin support [CUE]: https://cuelang.org [SOPS]: https://github.com/mozilla/sops -[kust]: https://kustomize.io/ [k8stut]: https://cuelang.org/docs/tutorials/ [eg]: examples/ [rel]: https://github.com/noris-network/cuegen/releases/latest -[kusteg]: examples/kustomize [cmp]: https://argo-cd.readthedocs.io/en/stable/user-guide/config-management-plugins/#option-2-configure-plugin-via-sidecar [cuegen-cmp]: https://hub.docker.com/r/nxcc/cuegen-cmp [expenv]: https://pkg.go.dev/os#ExpandEnv diff --git a/examples/kustomize/cuegen.yaml b/examples/kustomize/cuegen.yaml deleted file mode 100644 index b1dc8cd..0000000 --- a/examples/kustomize/cuegen.yaml +++ /dev/null @@ -1,8 +0,0 @@ -## kustomize plugin header -apiVersion: noris.net/mcs/v1beta1 -kind: Cuegen -metadata: - name: some-non-empty-value-required-by-kustomize - -## regular cuegen config -objectsPath: objects diff --git a/examples/kustomize/greeting.txt b/examples/kustomize/greeting.txt deleted file mode 100644 index 70163c8..0000000 --- a/examples/kustomize/greeting.txt +++ /dev/null @@ -1 +0,0 @@ -Hello from kustomize! \ No newline at end of file diff --git a/examples/kustomize/kustomization.yaml b/examples/kustomize/kustomization.yaml deleted file mode 100644 index f160731..0000000 --- a/examples/kustomize/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -generators: - - cuegen.yaml diff --git a/examples/kustomize/main.cue b/examples/kustomize/main.cue deleted file mode 100644 index 02990fe..0000000 --- a/examples/kustomize/main.cue +++ /dev/null @@ -1,13 +0,0 @@ -package kube - -configMap: [ID=_]: { - apiVersion: "v1" - kind: "ConfigMap" - metadata: name: ID -} - -configMap: "hello": data: { - greeting: string @readfile(greeting.txt) -} - -objects: [ for v in configMap {v}] diff --git a/examples/kustomize/readme.md b/examples/kustomize/readme.md deleted file mode 100644 index 4816ae8..0000000 --- a/examples/kustomize/readme.md +++ /dev/null @@ -1,17 +0,0 @@ -## Kustomize plugin - -If required, `cuegen` can be run as [kustomize][kust] plugin. -Currently only the deprecated ["exec plugin"][plug] method is supported. - -When running on linux and `cuegen` is already in your `PATH`, just run: - - PLUGIN_DIR=${XDG_CONFIG_HOME:=$HOME/.config}/kustomize/plugin/noris.net/mcs/v1beta1/cuegen - mkdir -p $PLUGIN_DIR - cp $(command -v cuegen) $PLUGIN_DIR/Cuegen - -To run this example `kustomize` has to be installed. Run the example with - - kustomize build --enable-alpha-plugins examples/kustomize - -[plug]: https://kubectl.docs.kubernetes.io/guides/extending_kustomize/exec_plugins/ -[kust]: https://kustomize.io/ diff --git a/examples/readme.md b/examples/readme.md index 0795057..96e5fd3 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -3,10 +3,10 @@ * [configmap](configmap): load external data into a ConfigMap * [values](values): load external structured data into a CUE value * [encrypted](encrypted): load and decrypt [SOPS][SOPS]-encrypted data -* [kustomize](kustomize): use `cuegen` as kustomize plugin * [components](components): compose several charts into one * [control-repository](control-repository): full example how to install several instances of an application in various environments +* [remote](remote): render chart from remote repository To run these examples, put `cuegen` into your `PATH` and execute diff --git a/examples/remote/readme.md b/examples/remote/readme.md new file mode 100644 index 0000000..80d19fe --- /dev/null +++ b/examples/remote/readme.md @@ -0,0 +1,11 @@ +## Load Remote charts + +As no local data is required, just run + + cuegen https://github.com/nxcc/cuegen-remote-test.git + +to render the chart located in the given repository. Environment Variables in the +given URL are expanded. Like in components, `ref` and `#`-prefixed subpaths can be used: + + cuegen "https://github.com/nxcc/cuegen-remote-test.git?ref=subpath#apps/app_b" + diff --git a/go.mod b/go.mod index 8a6c57a..3aebee4 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.20 require ( cuelang.org/go v0.5.0 - github.com/cue-exp/cueconfig v0.0.1 github.com/forensicanalysis/gitfs v0.2.1 github.com/go-git/go-git/v5 v5.7.0 github.com/joho/godotenv v1.5.1 + github.com/nxcc/cueconfig v0.0.1 github.com/rogpeppe/go-internal v1.10.0 go.mozilla.org/sops/v3 v3.7.3 golang.org/x/exp v0.0.0-20230202163644-54bba9f4231b diff --git a/go.sum b/go.sum index e6fd180..528293a 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,6 @@ github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b80 github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw= github.com/containerd/continuity v0.2.2 h1:QSqfxcn8c+12slxwu00AtzXrsami0MJb/MQs9lOLHLA= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cue-exp/cueconfig v0.0.1 h1:p8CZgrs3Ion58Agwyl2rYrguy/zNDS7vnBPL1TZOL8k= -github.com/cue-exp/cueconfig v0.0.1/go.mod h1:NmMZe8inhcWk4gc0iCpDBQd9slapQawwApS3n2XF/P4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -281,6 +279,8 @@ github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de h1:D5x39vF5KCwKQaw+OC9 github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de/go.mod h1:kJun4WP5gFuHZgRjZUWWuH1DTxCtxbHDOIJsudS8jzY= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxcc/cueconfig v0.0.1 h1:rnuTlLODSnZNrH84TJW1DldeaAA/jTLZiRg7WryhQR0= +github.com/nxcc/cueconfig v0.0.1/go.mod h1:KWh7yngeWWLCR2GShDV2P47m+25fjBVEgfGsZYZJ9AI= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= diff --git a/internal/app/main.go b/internal/app/main.go index 0172e69..1d54f8e 100644 --- a/internal/app/main.go +++ b/internal/app/main.go @@ -20,13 +20,15 @@ import ( "flag" "fmt" "io" + "io/fs" "log" "os" "path/filepath" "runtime/debug" + "strings" - "github.com/cue-exp/cueconfig" "github.com/noris-network/cuegen/internal/cuegen" + "github.com/nxcc/cueconfig" "gopkg.in/yaml.v3" ) @@ -34,7 +36,6 @@ const defaultYamlCuegenFile = "cuegen.yaml" const defaultCueCuegenFile = "cuegen.cue" var Build = "" -var runningAsKustomizePlugin = os.Getenv("KUSTOMIZE_PLUGIN_CONFIG_ROOT") != "" func Main() int { @@ -92,16 +93,7 @@ func Main() int { } // setup paths - chartRoot := "" - if runningAsKustomizePlugin { - cwd, err := os.Getwd() - if err != nil { - log.Fatalf("getwd: %v", err) - } - chartRoot = cwd - } else { - chartRoot = filepath.Dir(configFile) - } + chartRoot := filepath.Dir(configFile) chartRoot, err = filepath.Abs(chartRoot) if err != nil { @@ -123,8 +115,24 @@ func Main() int { // loadConfig loads the cuegen config. When a directory is passed, cuegen will // look for the default "cuegen.yaml" in that directory. func loadConfig(path string) (string, cuegen.Config, error) { + + var rootfs fs.FS + + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + path = os.ExpandEnv(path) + gitfs, err := cuegen.GetGitFS(path) + if err != nil { + return "", cuegen.Config{}, fmt.Errorf("in loadConfig: %v", err) + } + rootfs = gitfs + path = "." + } else { + rootfs = os.DirFS(".") + path = strings.TrimRight(path, "/") + } + file, err := func() (string, error) { - fileInfo, err := os.Stat(path) + fileInfo, err := fs.Stat(rootfs, path) if err != nil { return "", fmt.Errorf("stat: %v", err) } @@ -134,13 +142,13 @@ func loadConfig(path string) (string, cuegen.Config, error) { if fileInfo.IsDir() { // cuegen.cue? file := filepath.Join(path, defaultCueCuegenFile) - fileInfo, err := os.Stat(file) + fileInfo, err := fs.Stat(rootfs, file) if err == nil && fileInfo.Mode().IsRegular() { return file, nil } // cuegen.yaml? file = filepath.Join(path, defaultYamlCuegenFile) - fileInfo, err = os.Stat(file) + fileInfo, err = fs.Stat(rootfs, file) if err == nil && fileInfo.Mode().IsRegular() { return file, nil } @@ -155,17 +163,17 @@ func loadConfig(path string) (string, cuegen.Config, error) { } ext := filepath.Ext(file) if ext == ".cue" { - return loadCueConfig(file) + return loadCueConfig(rootfs, file) } - if ext == ".yml" || ext == ".yaml" || runningAsKustomizePlugin { - return loadYamlConfig(file) + if ext == ".yml" || ext == ".yaml" { + return loadYamlConfig(rootfs, file) } return "", cuegen.Config{}, errors.New("no config found") } // loadYamlConfig loads the cuegen config -func loadYamlConfig(file string) (string, cuegen.Config, error) { - fh, err := os.Open(file) +func loadYamlConfig(fsys fs.FS, file string) (string, cuegen.Config, error) { + fh, err := fsys.Open(file) if err != nil { return "", cuegen.Config{}, err } @@ -174,10 +182,11 @@ func loadYamlConfig(file string) (string, cuegen.Config, error) { decoder.KnownFields(true) if err := decoder.Decode(&config); err != nil { if errors.Is(err, io.EOF) { - return file, cuegen.Config{}, nil + return file, cuegen.Config{RootFS: &fsys}, nil } return "", cuegen.Config{}, err } + config.RootFS = &fsys return file, config, nil } @@ -185,10 +194,11 @@ func loadYamlConfig(file string) (string, cuegen.Config, error) { var cuegenConfigSchema []byte // loadCueConfig loads the cuegen config -func loadCueConfig(file string) (string, cuegen.Config, error) { +func loadCueConfig(fsys fs.FS, file string) (string, cuegen.Config, error) { config := struct{ Cuegen cuegen.Config }{} - if err := cueconfig.Load(file, cuegenConfigSchema, nil, nil, &config); err != nil { + if err := cueconfig.LoadFS(fsys, file, cuegenConfigSchema, nil, nil, &config); err != nil { return "", cuegen.Config{}, fmt.Errorf("load cue: %v", err) } + config.Cuegen.RootFS = &fsys return file, config.Cuegen, nil } diff --git a/internal/cuegen/main.go b/internal/cuegen/main.go index 9f6eb4d..1b64da3 100644 --- a/internal/cuegen/main.go +++ b/internal/cuegen/main.go @@ -46,6 +46,7 @@ type Config struct { ChartRoot string CheckPath string `yaml:"checkPath"` CheckPaths []string `yaml:"checkPaths"` + RootFS *fs.FS } type Component struct { @@ -65,11 +66,16 @@ type Cuegen struct { CheckPaths []string ChartRoot string DumpOverlays bool + RootFS *fs.FS } // Exec initializes the Cuegen struct and executes cuegen func Exec(config Config) error { + if os.Getenv("CUEGEN_DEBUG") == "true" { + config.Debug = true + } + // apply preferred defaults when all required fields are empty if config.ObjectsPath == "" && config.SecretDataPath == "" && @@ -88,6 +94,7 @@ func Exec(config Config) error { ChartRoot: config.ChartRoot, SecretDataPath: config.SecretDataPath, DumpOverlays: config.DumpOverlays, + RootFS: config.RootFS, } if config.CheckPath != "" { @@ -105,6 +112,13 @@ func Exec(config Config) error { } cg.Components = components + if cg.RootFS != nil { + cg.Components["remoteRootFS"] = Component{ + Filesystem: *cg.RootFS, + Type: "remoterootfs", + } + } + if cg.Debug { cg.PrintConfig() } @@ -115,12 +129,13 @@ func Exec(config Config) error { // getComponents loads components from the given paths func (cg Cuegen) getComponents(componentPaths []string) (Components, error) { components := Components{} + for _, componentPath := range componentPaths { component := Component{Path: componentPath, ID: generateID(componentPath)} switch { case strings.HasPrefix(componentPath, "http://") || strings.HasPrefix(componentPath, "https://") || strings.HasPrefix(componentPath, "git@"): - gitfs, err := getGitFS(componentPath) + gitfs, err := GetGitFS(componentPath) if err != nil { return nil, fmt.Errorf("getComponents: %v", err) } @@ -158,11 +173,11 @@ func generateID(name string) string { return base32.StdEncoding.EncodeToString(bs[:])[:10] } -// getGitFS returns a fs.FS from the given git repository URL -func getGitFS(component string) (fs.FS, error) { +// GetGitFS returns a fs.FS from the given git repository URL +func GetGitFS(component string) (fs.FS, error) { uri, ref, subDir, err := parseGitURL(component) if err != nil { - return nil, fmt.Errorf("getGitURL: open: %v", err) + return nil, fmt.Errorf("GetGitURL: open: %v", err) } opts := git.CloneOptions{ diff --git a/scripts/test-all-examples.sh b/scripts/test-all-examples.sh index 7c67710..ffab284 100755 --- a/scripts/test-all-examples.sh +++ b/scripts/test-all-examples.sh @@ -35,14 +35,9 @@ cuegen control-repository/control/prod-cluster/wekan-prod/ | grep -q "namespace: cuegen control-repository/control/prod-cluster/wekan-qa/ | grep -q "namespace: cuegen-demo-qa" echo " OK" -echo kustomize plugin -tempdir=$(mktemp -d) -XDG_CONFIG_HOME=$tempdir -export XDG_CONFIG_HOME -cuegen_dir="$XDG_CONFIG_HOME/kustomize/plugin/noris.net/mcs/v1beta1/cuegen" -mkdir -p "$cuegen_dir" -cp "$(command -v cuegen)" "$cuegen_dir/Cuegen" -kustomize build --enable-alpha-plugins kustomize | grep -q "Hello from kustomize" +echo remote +cuegen https://github.com/nxcc/cuegen-remote-test.git | grep -q 'field1: test text 123' +cuegen "https://github.com/nxcc/cuegen-remote-test.git?ref=subpath#apps/app_b" | grep -q 'field1: test yaml 5678' echo " OK" # done diff --git a/tests/local/kustomize.txtar b/tests/local/kustomize.txtar deleted file mode 100644 index cc47605..0000000 --- a/tests/local/kustomize.txtar +++ /dev/null @@ -1,47 +0,0 @@ -### kustomize.txtar - -# is kustomize installed? -[!exec:kustomize] skip # kustomize not found - -# started from go test? -[exec:started_from_go_test] skip # skip, use testscript instead - -# run cuegen -exec cuegen chart -stdout 'aaa: FILE' - -# install cuegen as kustomize plugin -mkdir $WORK/noris.net/mcs/v1beta1/cuegen -exec sh -c 'ln -s $(which cuegen) $WORK/noris.net/mcs/v1beta1/cuegen/Cuegen' - -# run kustomize -env KUSTOMIZE_PLUGIN_HOME=$WORK -exec kustomize build --enable-alpha-plugins chart -stdout 'aaa: FILE' - --- chart/kustomization.yaml -- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -generators: - - cuegen.yaml - --- chart/cuegen.yaml -- -apiVersion: noris.net/mcs/v1beta1 -kind: Cuegen -metadata: - name: config -objectsPath: objects - --- chart/a.cue -- -package kube - -obj: { - metadata: name: "dummy" // name and kind are - kind: "ConfigMap" // required by kustomize - data: aaa: string @readfile(file=trim) -} - -objects: [obj] - --- chart/file -- -FILE diff --git a/tests/remote/remote-root.txtar b/tests/remote/remote-root.txtar new file mode 100644 index 0000000..3270951 --- /dev/null +++ b/tests/remote/remote-root.txtar @@ -0,0 +1,32 @@ +### remote-root.txtar + +# remote +exec cuegen https://github.com/nxcc/cuegen-remote-test.git +cmp stdout expect1 + +# remote with subpath and ref +exec cuegen 'https://github.com/nxcc/cuegen-remote-test.git?ref=subpath#apps/app_a' +cmp stdout expect2 + +# remote with subpath, ref and cuegen.yaml +exec cuegen 'https://github.com/nxcc/cuegen-remote-test.git?ref=subpath#apps/app_b' +cmp stdout expect3 + +-- expect1 -- +data: + field1: test text 123 + field2: value is 'some value' +metadata: + name: myconfig +-- expect2 -- +data: + field1: test text 1234 + field2: value is 'app-a' +metadata: + name: myconfig +-- expect3 -- +data: + field1: test yaml 5678 + field2: value is 'app-b' +metadata: + name: myconfig