buildsys: extend external-files to vendor go modules

packages/hotdog/Cargo.toml

@@ -11,24 +11,7 @@ path = "pkg.rs"
 [[package.metadata.build-package.external-files]]
 url = "https://github.com/bottlerocket-os/hotdog/archive/b85b75576adbbd7e133b54d71ebc11a28acf40db/hotdog-b85b755.tar.gz"
 sha512 = "9b2d5cb0e25d774d11dd6eb577e07af85f36fcd6e816b9df88d7ca1da273695f15ce6831026d28e68355512a07d0ac673b5ce9771d969f1c5ca4f14bc631deb8"
-
-[[package.metadata.build-package.external-files]]
-url = "https://github.com/opencontainers/runtime-spec/archive/v1.0.2/runtime-spec-1.0.2.tar.gz"
-sha512 = "96676b702d02409d33a5c81886c4db4bf45283c628933c6f0f4c2ed0d7cc44fafe95249151d7dc2d1fc5225944d172cdb45fc2f2f5f9bb87531e93421933b664"
-
-[[package.metadata.build-package.external-files]]
-url = "https://github.com/golang/sys/archive/8c9f86f7a55f5af45a9080f6ed3ac915382d369c/sys-8c9f86f.tar.gz"
-sha512 = "054dfba40563e2537dffcd8464debf44bcb71d5790603f21a324ec940d7018d8161d50d535471f5ffeada9647ef390d599d567dd0515eeaa677103c60e502b40"
-
-[[package.metadata.build-package.external-files]]
-path = "go-selinux-v1.10.1.tar.gz"
-url = "https://github.com/opencontainers/selinux/archive/refs/tags/v1.10.1.tar.gz"
-sha512 = "f79af22c28ac14b3ca93c1c60fc6f986ec6b22c0f2d77fc4192b95e0a497798acfe9be48f4e162e0a31c7e4c3c78470bdee6faab1c06f8688b51a0c20331b77a"
-
-[[package.metadata.build-package.external-files]]
-path = "libcap-v1.2.63.tar.gz"
-url = "https://git.kernel.org/pub/scm/libs/libcap/libcap.git/snapshot/libcap-cap/v1.2.63.tar.gz"
-sha512 = "f88f85ce5849c0c6a519c3f269a5b221a66b98d30abfa15a69a760ba85d84b5cf89f41ad1912fc0643a5533b44f0b2d09597229b61f4d08473ad6124e40bc864"
+bundle-modules = [ "go" ]
 
 [build-dependencies]
 glibc = { path = "../glibc" }

packages/hotdog/hotdog.spec

@@ -10,67 +10,34 @@
 %global gitrev b85b75576adbbd7e133b54d71ebc11a28acf40db
 %global shortrev %(c=%{gitrev}; echo ${c:0:7})
 
-%global gosysrev 8c9f86f7a55f5af45a9080f6ed3ac915382d369c
-%global gosysrevshort %(c=%{gosysrev}; echo ${c:0:7})
-
-%global runtimespec 1.0.2
-
-%global goselinux 1.10.1
-
-%global libcap 1.2.63
-
 Name: %{_cross_os}hotdog
 Version: 1.0.1
 Release: 1%{?dist}
 Summary: Tool with OCI hooks to run the Log4j Hot Patch in containers
 License: Apache-2.0
 URL: https://github.com/bottlerocket-os/hotdog
-Source0: https://%{goimport}/archive/%{gorev}/%{gorepo}-%{shortrev}.tar.gz
-Source1: https://github.com/opencontainers/runtime-spec/archive/v%{runtimespec}/runtime-spec-%{runtimespec}.tar.gz
-Source2: https://github.com/golang/sys/archive/%{gosysrev}/sys-%{gosysrevshort}.tar.gz
-Source3: https://github.com/opencontainers/selinux/archive/refs/tags/v%{goselinux}.tar.gz#/go-selinux-v%{goselinux}.tar.gz
-Source4: https://git.kernel.org/pub/scm/libs/libcap/libcap.git/snapshot/libcap-cap/v%{libcap}.tar.gz#/libcap-v%{libcap}.tar.gz
-
+Source0: %{gorepo}-%{shortrev}.tar.gz
+Source1: bundled-%{gorepo}-%{shortrev}.tar.gz
 BuildRequires: %{_cross_os}glibc-devel
 Requires: %{_cross_os}log4j2-hotpatch
 
 %description
 %{summary}.
 
 %prep
-%autosetup -Sgit -n %{gorepo}-%{gitrev} -p1
-%cross_go_setup %{gorepo}-%{gitrev} %{goproject} %{goimport}
-
-# We need to manage these third-party dependencies because the hotdog
-# "release" that we use doesn't include the `vendor` directory, unlike our other
-# go third party dependencies
-mkdir -p GOPATH/src/github.com/opencontainers/runtime-spec
-tar -C GOPATH/src/github.com/opencontainers/runtime-spec -xzf %{SOURCE1} --strip 1
-cp GOPATH/src/github.com/opencontainers/runtime-spec/LICENSE LICENSE.runtime-spec
-
-mkdir -p GOPATH/src/golang.org/x/sys
-tar -C GOPATH/src/golang.org/x/sys -xzf %{SOURCE2} --strip 1
-cp GOPATH/src/golang.org/x/sys/LICENSE LICENSE.golang-sys
-
-mkdir -p GOPATH/src/github.com/opencontainers/selinux
-tar -C GOPATH/src/github.com/opencontainers/selinux -xzf %{SOURCE3} --strip 1
-cp GOPATH/src/github.com/opencontainers/selinux/LICENSE LICENSE.go-selinux
-
-mkdir -p GOPATH/src/kernel.org/pub/linux/libs/security/libcap
-tar -C GOPATH/src/kernel.org/pub/linux/libs/security/libcap -xzf %{SOURCE4} --strip 2
-cp GOPATH/src/kernel.org/pub/linux/libs/security/libcap/License LICENSE.libcap
+%setup -n %{gorepo}-%{gitrev}
+%setup -T -D -n %{gorepo}-%{gitrev} -b 1
 
 %build
-%cross_go_configure %{goimport}
+%set_cross_go_flags
 
 # Set CGO_ENABLED=0 to statically link hotdog-hotpath, since it runs inside containers that
 # may not have the glibc version used to compile it
-# Set `GO111MODULE=off` to force golang to look for the dependencies in the GOPATH
-CGO_ENABLED=0 GO111MODULE=off go build -installsuffix cgo -a -ldflags "-s" -o hotdog-hotpatch ./cmd/hotdog-hotpatch
+CGO_ENABLED=0 go build ${GOFLAGS} -installsuffix cgo -a -ldflags "-s" -o hotdog-hotpatch ./cmd/hotdog-hotpatch
 
 # The oci hooks commands can be compiled as we usually compile golang packages
 for cmd in hotdog-cc-hook hotdog-poststart-hook; do
-  GO111MODULE=off go build -buildmode=pie -ldflags "${GOLDFLAGS}" -o $cmd ./cmd/$cmd
+  go build ${GOFLAGS} -buildmode=pie -ldflags "${GOLDFLAGS}" -o $cmd ./cmd/$cmd
 done
 
 %install
@@ -83,9 +50,12 @@ for cmd in hotdog-cc-hook hotdog-poststart-hook; do
   install -p -m 0755 $cmd %{buildroot}%{_cross_libexecdir}/hotdog
 done
 
+%cross_scan_attribution go-vendor vendor
+
 %files
-%license LICENSE LICENSE.runtime-spec LICENSE.golang-sys LICENSE.go-selinux LICENSE.libcap
+%license LICENSE
 %{_cross_attribution_file}
+%{_cross_attribution_vendor_dir}
 %dir %{_cross_libexecdir}/hotdog
 %dir %{_cross_datadir}/hotdog
 %{_cross_libexecdir}/hotdog/hotdog-cc-hook

packages/oci-add-hooks/Cargo.toml

@@ -11,14 +11,7 @@ path = "pkg.rs"
 [[package.metadata.build-package.external-files]]
 url = "https://github.com/awslabs/oci-add-hooks/archive/ef29fe312d2e1858d5eb28ab0abe0cbee298a165/oci-add-hooks-ef29fe3.tar.gz"
 sha512 = "018b561f838172e768a70acdeb2c27939f931391ced019a23c5193eee6b8970bc02a3e5fa05917010ca2064d1876649ef139d7657700c42a3ddd6e2c174f27dc"
-
-[[package.metadata.build-package.external-files]]
-url = "https://github.com/bitly/go-simplejson/archive/v0.5.0/go-simplejson-0.5.0.tar.gz"
-sha512 = "39c0d85d6ee06a8a795c1e344f0669f5ae8371d1122f09a1b13e5ff7629dd7faf633f9fcb449e19aadab9ad3e42e93143205781a822a29f27758872cf7e09e18"
-
-[[package.metadata.build-package.external-files]]
-url = "https://github.com/joeshaw/json-lossless/archive/e0cd1ca6349bf167e33d44f28c14c728a277205f/json-lossless-e0cd1ca.tar.gz"
-sha512 = "b9eb6170f662a396370ae1e170d89e15efc0a96fee6046fbd749c7a65f09f808e08bc2cf91962db65fd86a2aac4dddf428412b568fe1d03a77a7de22ad0690aa"
+bundle-modules = [ "go" ]
 
 [build-dependencies]
 glibc = { path = "../glibc" }

packages/oci-add-hooks/oci-add-hooks.spec

@@ -4,48 +4,37 @@
 
 %global gitrev ef29fe312d2e1858d5eb28ab0abe0cbee298a165
 %global shortrev %(c=%{gitrev}; echo ${c:0:7})
-%global gosimplejson 0.5.0
-%global jsonlosslessrev e0cd1ca6349bf167e33d44f28c14c728a277205f
-%global jsonlosslessshort %(c=%{jsonlosslessrev}; echo ${c:0:7})
 
 Name: %{_cross_os}oci-add-hooks
 Version: 1.0.0
 Release: 1%{?dist}
 Summary: OCI runtime wrapper that injects OCI hooks
 License: Apache-2.0 and MIT
 URL: https://github.com/awslabs/oci-add-hooks
-Source0: https://%{goimport}/archive/%{gorev}/%{gorepo}-%{shortrev}.tar.gz
-Source1: https://github.com/bitly/go-simplejson/archive/v%{gosimplejson}/go-simplejson-%{gosimplejson}.tar.gz
-Source2: https://github.com/joeshaw/json-lossless/archive/%{jsonlosslessrev}/json-lossless-%{jsonlosslessshort}.tar.gz
-
+Source0: %{gorepo}-%{shortrev}.tar.gz
+Source1: bundled-%{gorepo}-%{shortrev}.tar.gz
 BuildRequires: %{_cross_os}glibc-devel
 
 %description
 %{summary}.
 
 %prep
-%autosetup -n %{gorepo}-%{gitrev}
-%cross_go_setup %{gorepo}-%{gitrev} %{goproject} %{goimport}
-
-# We need to manage these third-party dependencies because the oci-add-hooks
-# "release" that we use doesn't include the `vendor` directory, unlike our other
-# go third party dependencies
-mkdir -p GOPATH/src/github.com/bitly/go-simplejson GOPATH/src/github.com/joeshaw/json-lossless
-tar -C GOPATH/src/github.com/bitly/go-simplejson -xzf %{SOURCE1} --strip 1
-cp GOPATH/src/github.com/bitly/go-simplejson/LICENSE LICENSE.go-simplejson
-tar -C GOPATH/src/github.com/joeshaw/json-lossless -xzf %{SOURCE2} --strip 1
-cp GOPATH/src/github.com/joeshaw/json-lossless/LICENSE LICENSE.json-lossless
+%setup -n %{gorepo}-%{gitrev}
+%setup -T -D -n %{gorepo}-%{gitrev} -b 1
 
 %build
-%cross_go_configure %{goimport}
-# We use `GO111MODULE=off` to force golang to look for the dependencies in the GOPATH
-GO111MODULE=off go build -v -x -buildmode=pie -ldflags="${GOLDFLAGS}" -o oci-add-hooks
+%set_cross_go_flags
+export LD_VERSION="-X main.commit=oci-add-hooks-%{gitrev}"
+go build ${GOFLAGS} -v -x -buildmode=pie -ldflags="${GOLDFLAGS} ${LD_VERSION}" -o oci-add-hooks
 
 %install
 install -d %{buildroot}%{_cross_bindir}
 install -p -m 0755 oci-add-hooks %{buildroot}%{_cross_bindir}
 
+%cross_scan_attribution go-vendor vendor
+
 %files
-%license LICENSE NOTICE LICENSE.go-simplejson LICENSE.json-lossless
+%license LICENSE NOTICE
 %{_cross_attribution_file}
+%{_cross_attribution_vendor_dir}
 %{_cross_bindir}/oci-add-hooks

tools/buildsys/src/gomod.rs

@@ -0,0 +1,199 @@
+/*!
+Packages using the Go programming language may have upstream tar archives that
+include only the source code of the project, but not the source code of any
+dependencies. The Go programming language promotes the use of "modules" for
+dependencies. Projects adopting modules will provide `go.mod` and `go.sum` files.
+
+This Rust module extends the functionality of `packages.metadata.build-package.external-files`
+and provides the ability to retrieve and validate dependencies
+declared using Go modules given a tar archive containing a `go.mod` and `go.sum`.
+
+The location where dependencies are retrieved from are controlled by the
+standard environment variables employed by the Go tool: `GOPROXY`, `GOSUMDB`, and
+`GOPRIVATE`. These variables are automatically retrieved from the host environment
+when the docker-go script is invoked.
+
+ */
+
+pub(crate) mod error;
+use error::Result;
+
+use super::manifest;
+use duct::cmd;
+use snafu::{ensure, OptionExt, ResultExt};
+use std::io::Write;
+use std::os::unix::fs::PermissionsExt;
+use std::path::{Path, PathBuf};
+use std::{env, fs};
+
+pub(crate) struct GoMod;
+
+const GO_MOD_DOCKER_SCRIPT_NAME: &str = "docker-go-script.sh";
+
+// The following bash template script is intended to be run within a container
+// using the docker-go tool found in this codebase under `tools/docker-go`.
+//
+// This script inspects the top level directory found in the package upstream
+// archive and uses that as the default Go module path if no explicit module
+// path was provided. It will then untar the archive, vendor the Go
+// dependencies, create a new archive using the {module-path}/vendor directory
+// and name it the output path provided. If no output path was given, it
+// defaults to "bundled-{package-file-name}". Finally, it cleans up by removing
+// the untar'd source code. The upstream archive remains intact and both tar
+// files can then be used during packaging.
+//
+// This script exists as an in memory template string literal and is populated
+// into a temporary file in the package directory itself to enable buildsys to
+// be as portable as possible and have no dependecy on runtime paths. Since
+// buildsys is executed from the context of many different package directories,
+// managing a temporary file via this Rust module prevents having to aquire the
+// path of some static script file on the host system.
+const GO_MOD_SCRIPT_TMPL: &str = r###"#!/bin/bash
+
+set -e
+
+toplevel=$(tar tf __LOCAL_FILE_NAME__ | head -1)
+if [ -z __MOD_DIR__ ] ; then
+    targetdir="${toplevel}"
+else
+    targetdir="__MOD_DIR__"
+fi
+
+tar xf __LOCAL_FILE_NAME__
+
+pushd "${targetdir}"
+    go list -mod=readonly ./... >/dev/null && go mod vendor
+popd
+
+tar czf __OUTPUT__ "${targetdir}"/vendor
+rm -rf "${targetdir}"
+"###;
+
+impl GoMod {
+    pub(crate) fn vendor(
+        root_dir: &Path,
+        package_dir: &Path,
+        external_file: &manifest::ExternalFile,
+    ) -> Result<()> {
+        let url_file_name = extract_file_name(&external_file.url)?;
+        let local_file_name = &external_file.path.as_ref().unwrap_or(&url_file_name);
+        ensure!(
+            local_file_name.components().count() == 1,
+            error::InputFileSnafu
+        );
+
+        let full_path = package_dir.join(local_file_name);
+        ensure!(
+            full_path.is_file(),
+            error::InputFileBadSnafu { path: full_path }
+        );
+
+        // If a module directory was not provided, set as an empty path.
+        // By default, without a provided module directory, tar will be passed
+        // the first directory found in the archives as the top level Go module
+        let default_empty_path = PathBuf::from("");
+        let mod_dir = external_file
+            .bundle_root_path
+            .as_ref()
+            .unwrap_or(&default_empty_path);
+
+        // Use a default "bundle-{name-of-file}" if no output path was provided
+        let default_output_path =
+            PathBuf::from(format!("bundled-{}", local_file_name.to_string_lossy()));
+        let output_path_arg = external_file
+            .bundle_output_path
+            .as_ref()
+            .unwrap_or(&default_output_path);
+        println!(
+            "cargo:rerun-if-changed={}",
+            output_path_arg.to_string_lossy()
+        );
+
+        // Our SDK and toolchain are picked by the external `cargo make` invocation.
+        let sdk = env::var("BUILDSYS_SDK_IMAGE").context(error::EnvironmentSnafu {
+            var: "BUILDSYS_SDK_IMAGE",
+        })?;
+
+        let args = DockerGoArgs {
+            module_path: package_dir,
+            sdk_image: sdk,
+            go_mod_cache: &root_dir.join(".gomodcache"),
+            command: format!("./{}", GO_MOD_DOCKER_SCRIPT_NAME),
+        };
+
+        // Create and/or write the temporary script file to the package directory
+        // using the script template string and placeholder variables
+        let script_contents = GO_MOD_SCRIPT_TMPL
+            .replace("__LOCAL_FILE_NAME__", &local_file_name.to_string_lossy())
+            .replace("__MOD_DIR__", &mod_dir.to_string_lossy())
+            .replace("__OUTPUT__", &default_output_path.to_string_lossy());
+        let script_path = format!(
+            "{}/{}",
+            package_dir.to_string_lossy(),
+            GO_MOD_DOCKER_SCRIPT_NAME
+        );
+        {
+            let mut script_file = fs::File::create(&script_path).unwrap();
+            fs::set_permissions(&script_path, fs::Permissions::from_mode(0o777)).unwrap();
+            script_file.write_all(script_contents.as_bytes()).unwrap();
+        }
+
+        let res = docker_go(root_dir, &args);
+        fs::remove_file(script_path).unwrap();
+        res
+    }
+}
+
+fn extract_file_name(url: &str) -> Result<PathBuf> {
+    let parsed = reqwest::Url::parse(url).context(error::InputUrlSnafu { url })?;
+    let name = parsed
+        .path_segments()
+        .context(error::InputFileBadSnafu { path: url })?
+        .last()
+        .context(error::InputFileBadSnafu { path: url })?;
+    Ok(name.into())
+}
+
+struct DockerGoArgs<'a> {
+    module_path: &'a Path,
+    sdk_image: String,
+    go_mod_cache: &'a Path,
+    command: String,
+}
+
+/// Run `docker-go` with the specified arguments.
+fn docker_go(root_dir: &Path, dg_args: &DockerGoArgs) -> Result<()> {
+    let args = vec![
+        "--module-path",
+        dg_args
+            .module_path
+            .to_str()
+            .context(error::InputFileSnafu)?,
+        "--sdk-image",
+        &dg_args.sdk_image,
+        "--go-mod-cache",
+        dg_args
+            .go_mod_cache
+            .to_str()
+            .context(error::InputFileSnafu)?,
+        "--command",
+        &dg_args.command,
+    ];
+    let arg_string = args.join(" ");
+    let program = root_dir.join("tools/docker-go");
+    println!("program: {}", program.to_string_lossy());
+    let output = cmd(program, args)
+        .stderr_to_stdout()
+        .stdout_capture()
+        .unchecked()
+        .run()
+        .context(error::CommandStartSnafu)?;
+
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    println!("{}", &stdout);
+    ensure!(
+        output.status.success(),
+        error::DockerExecutionSnafu { args: arg_string }
+    );
+    Ok(())
+}