Use AL2023 for integ test container

This change moves our integ test container from Alpine to AL2023. The
motivation is to be able to test features of a bigger OS, such as
testing systemd socket activation.

Signed-off-by: Kern Walster <walster@amazon.com>

Dockerfile

@@ -18,7 +18,7 @@ ARG NERDCTL_VERSION=1.7.1
 
 FROM public.ecr.aws/docker/library/registry:3.0.0-alpha.1 AS registry
 
-FROM public.ecr.aws/docker/library/golang:1.22.7-alpine AS containerd-snapshotter-base
+FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS containerd-snapshotter-base
 
 ARG CONTAINERD_VERSION
 ARG RUNC_VERSION
@@ -29,16 +29,15 @@ ENV GOCOVERDIR /test_coverage
 
 COPY ./integ_entrypoint.sh /integ_entrypoint.sh
 COPY . $GOPATH/src/github.com/awslabs/soci-snapshotter
-RUN apk update && apk upgrade
-RUN apk add --no-cache \
-    btrfs-progs-libs \
-    curl \
-    fuse \
-    gcc \
-    libc6-compat \
-    libseccomp-dev \
+RUN dnf update && dnf upgrade && dnf install -y \
+    diffutils \
+    findutils \
+    gzip \
+    iptables \
     pigz \
-    zlib-dev
+    procps \
+    tar \
+    util-linux-core
 RUN cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/soci /usr/local/bin/ && \
     cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/soci-snapshotter-grpc /usr/local/bin/ && \
     mkdir /etc/soci-snapshotter-grpc && \

integration/pull_test.go

@@ -734,13 +734,13 @@ host = "%s"
 insecure = true
 `, regConfig.host, regAltConfig.hostWithPort())
 
+	crtPath := filepath.Join(caCertDir, "domain.crt")
 	// Setup environment
-	if err := testutil.WriteFileContents(sh, filepath.Join(caCertDir, "domain.crt"), crt, 0600); err != nil {
+	if err := testutil.WriteFileContents(sh, crtPath, crt, 0600); err != nil {
 		t.Fatalf("failed to write %v: %v", caCertDir, err)
 	}
 	sh.
-		X("apk", "add", "--no-cache", "iptables").
-		X("update-ca-certificates").
+		X("trust", "anchor", crtPath).
 		Retry(100, "nerdctl", "login", "-u", regConfig.user, "-p", regConfig.pass, regConfig.host)
 
 	imageName := rabbitmqImage

integration/run_test.go

@@ -306,7 +306,6 @@ disable = true
 			rebootContainerd(t, sh, getContainerdConfigToml(t, false), getSnapshotterConfigToml(t, false, config))
 			// Re-pull image from our local registry mirror
 			sh.X(append(imagePullCmd, "--soci-index-digest", indexDigest, regConfig.mirror(containerImage).ref)...)
-			sh.X("apk", "add", "--no-cache", "--quiet", "iptables")
 
 			containerRunCmd := append(runSociCmd, image, "cat", "/etc/hosts")
 

integration/util_test.go

@@ -569,11 +569,12 @@ networks:
 	sh = shell.New(de, testutil.NewTestingReporter(t))
 
 	// Install cert and login to the registry
-	if err := testutil.WriteFileContents(sh, filepath.Join(caCertDir, "domain.crt"), crt, 0600); err != nil {
+	crtPath := filepath.Join(caCertDir, "domain.crt")
+	if err := testutil.WriteFileContents(sh, crtPath, crt, 0600); err != nil {
 		t.Fatalf("failed to write cert at %v: %v", caCertDir, err)
 	}
 	sh.
-		X("update-ca-certificates").
+		X("trust", "anchor", crtPath).
 		Retry(100, "nerdctl", "login", "-u", r.user, "-p", r.pass, r.host)
 	return sh, func() error {
 		killErr := testutil.KillMatchingProcess(sh, "soci-snapshotter-grpc")
@@ -585,7 +586,7 @@ networks:
 				return errors.Join(killErr, err)
 			}
 		}
-		return nil
+		return killErr
 	}
 }
 
@@ -722,8 +723,14 @@ func rebootContainerd(t *testing.T, sh *shell.Shell, customContainerdConfig, cus
 	)
 
 	// cleanup directories
-	testutil.KillMatchingProcess(sh, "containerd")
-	testutil.KillMatchingProcess(sh, "soci-snapshotter-grpc")
+	err := testutil.KillMatchingProcess(sh, "containerd")
+	if err != nil {
+		sh.Fatal("failed to kill containerd: %v", err)
+	}
+	err = testutil.KillMatchingProcess(sh, "soci-snapshotter-grpc")
+	if err != nil {
+		sh.Fatal("failed to kill soci: %v", err)
+	}
 	removeDirContents(sh, containerdRoot)
 	if isDirExists(sh, containerdStatus) {
 		removeDirContents(sh, containerdStatus)

util/dockershell/shell.go

@@ -125,7 +125,7 @@ func New(de *dexec.Exec, r Reporter) *Shell {
 	}
 }
 
-func (s *Shell) fatal(format string, v ...interface{}) *Shell {
+func (s *Shell) Fatal(format string, v ...interface{}) *Shell {
 	s.r.Errorf(format, v...)
 	s.err = fmt.Errorf(format, v...)
 	s.invalidMu.Lock()
@@ -161,14 +161,14 @@ func (s *Shell) X(args ...string) *Shell {
 		return s
 	}
 	if len(args) < 1 {
-		return s.fatal("no command to run")
+		return s.Fatal("no command to run")
 	}
 	s.r.Logf(">>> Running: %v\n", args)
 	cmd := s.Command(args[0], args[1:]...)
 	cmd.Stdout = s.r.Stdout()
 	cmd.Stderr = s.r.Stderr()
 	if err := cmd.Run(); err != nil {
-		return s.fatal("failed to run %v: %v", args, err)
+		return s.Fatal("failed to run %v: %v", args, err)
 	}
 	return s
 }
@@ -180,7 +180,7 @@ func (s *Shell) XLog(args ...string) *Shell {
 		return s
 	}
 	if len(args) < 1 {
-		return s.fatal("no command to run")
+		return s.Fatal("no command to run")
 	}
 	s.r.Logf(">>> Running: %v\n", args)
 	cmd := s.Command(args[0], args[1:]...)
@@ -200,7 +200,7 @@ func (s *Shell) Gox(args ...string) *Shell {
 		return s
 	}
 	if len(args) < 1 {
-		return s.fatal("no command to run")
+		return s.Fatal("no command to run")
 	}
 	go func() {
 		s.r.Logf(">>> Running: %v\n", args)
@@ -234,7 +234,7 @@ func (s *Shell) Pipe(out io.Writer, commands ...[]string) *Shell {
 	for i, args := range commands {
 		i, args := i, args
 		if len(args) < 1 {
-			return s.fatal("no command to run")
+			return s.Fatal("no command to run")
 		}
 		s.r.Logf(">>> Running: %v\n", args)
 		cmd := s.Command(args[0], args[1:]...)
@@ -271,7 +271,7 @@ func (s *Shell) Pipe(out io.Writer, commands ...[]string) *Shell {
 		}
 	}
 	if !ok {
-		return s.fatal("could not run %v", commands)
+		return s.Fatal("could not run %v", commands)
 	}
 
 	return s
@@ -296,7 +296,7 @@ func (s *Shell) Retry(num int, args ...string) *Shell {
 		s.r.Logf("failed to run (%d/%d) %v: %v", i, num, args, err)
 		time.Sleep(time.Second)
 	}
-	return s.fatal("failed to run %v", args)
+	return s.Fatal("failed to run %v", args)
 }
 
 // O executes a command and return the stdout. Stderr is streamed to Reporter. When the command fails,
@@ -307,15 +307,15 @@ func (s *Shell) O(args ...string) []byte {
 		return nil
 	}
 	if len(args) < 1 {
-		s.fatal("no command to run")
+		s.Fatal("no command to run")
 		return nil
 	}
 	s.r.Logf(">>> Getting output of: %v\n", args)
 	cmd := s.Command(args[0], args[1:]...)
 	cmd.Stderr = s.r.Stderr()
 	out, err := cmd.Output()
 	if err != nil {
-		s.fatal("failed to run for getting output from %v: %v", args, err)
+		s.Fatal("failed to run for getting output from %v: %v", args, err)
 		return nil
 	}
 	return out
@@ -328,7 +328,7 @@ func (s *Shell) OLog(args ...string) ([]byte, error) {
 		return nil, fmt.Errorf("invalid shell")
 	}
 	if len(args) < 1 {
-		s.fatal("no command to run")
+		s.Fatal("no command to run")
 		return nil, fmt.Errorf("no command to run")
 	}
 	s.r.Logf(">>> Getting output of: %v\n", args)

util/testutil/shell.go

@@ -340,9 +340,9 @@ func CopyInDir(sh *shell.Shell, from, to string) error {
 // KillMatchingProcess kills processes that "ps" line matches the specified pattern in the
 // specified execution environment.
 func KillMatchingProcess(sh *shell.Shell, psLinePattern string) error {
-	data, err := sh.Command("ps", "auxww").Output()
+	data, err := sh.Command("ps", "axo", "pid,command").Output()
 	if err != nil {
-		return fmt.Errorf("failed to run ps command : %v", err)
+		return fmt.Errorf("failed to run ps command: %v", err)
 	}
 	var targets []int
 	scanner := bufio.NewScanner(bytes.NewReader(data))
@@ -367,14 +367,33 @@ func KillMatchingProcess(sh *shell.Shell, psLinePattern string) error {
 
 	var allErr error
 	for _, pid := range targets {
-		// Send SIGTERM so the unit under test correctly writes integration coverage reports to Go coverage directory.
-		if err := sh.Command("kill", "-2", fmt.Sprintf("%d", pid)).Run(); err != nil {
-			errors.Join(allErr, fmt.Errorf("failed to kill %v: %w", pid, err))
-		}
+		allErr = errors.Join(allErr, killProcess(sh, pid))
+
 	}
 	return allErr
 }
 
+func killProcess(sh *shell.Shell, pid int) error {
+	ex := sh.Command("kill", "-2", fmt.Sprintf("%d", pid))
+	r, err := ex.StderrPipe()
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+	// Send SIGTERM so the unit under test correctly writes integration coverage reports to Go coverage directory.
+	if err := ex.Run(); err != nil {
+		stderr, err := io.ReadAll(r)
+		if err != nil {
+			return err
+		}
+		// If the process disappeared between the ps and the kill, don't treat it as an error
+		if !strings.Contains(string(stderr), "No such process") {
+			return err
+		}
+	}
+	return nil
+}
+
 func RemoveContentStoreContent(sh *shell.Shell, contentStoreType store.ContentStoreType, contentDigest string) error {
 	contentStoreType, err := store.CanonicalizeContentStoreType(contentStoreType)
 	if err != nil {