feat: support bootstrap buildkit with HTTP (#1652)

Signed-off-by: Keming <kemingyang@tensorchord.ai>
tensorchord · Jun 14, 2023 · 56406e9 · 56406e9
1 parent 936b0ec
commit 56406e9
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 22 deletions.
diff --git a/pkg/app/bootstrap.go b/pkg/app/bootstrap.go
@@ -65,6 +65,11 @@ var CommandBootstrap = &cli.Command{
 				sshconfig.GetPublicKeyOrPanic(), sshconfig.GetPrivateKeyOrPanic()),
 			Aliases: []string{"k"},
 		},
+		&cli.BoolFlag{
+			Name:  "use-http",
+			Usage: "Use HTTP instead of HTTPS for the registry",
+			Value: false,
+		},
 	},
 
 	Action: bootstrap,
@@ -279,17 +284,23 @@ func buildkit(clicontext *cli.Context) error {
 
 	logrus.Debug("bootstrap the buildkitd container")
 	var bkClient buildkitd.Client
+	mirror := clicontext.String("dockerhub-mirror")
+	setRegistryCA := clicontext.IsSet("registry-ca-keypair")
+	useHTTP := clicontext.Bool("use-http")
+
+	if setRegistryCA && useHTTP {
+		return errors.New("cannot use both registry CA and HTTP")
+	}
+
 	if c.Builder == types.BuilderTypeMoby {
 		bkClient, err = buildkitd.NewMobyClient(clicontext.Context,
-			c.Builder, c.BuilderAddress, clicontext.String("dockerhub-mirror"),
-			clicontext.IsSet("registry-ca-keypair"))
+			c.Builder, c.BuilderAddress, mirror, setRegistryCA, useHTTP)
 		if err != nil {
 			return errors.Wrap(err, "failed to create moby buildkit client")
 		}
 	} else {
 		bkClient, err = buildkitd.NewClient(clicontext.Context,
-			c.Builder, c.BuilderAddress, clicontext.String("dockerhub-mirror"),
-			clicontext.IsSet("registry-ca-keypair"))
+			c.Builder, c.BuilderAddress, mirror, setRegistryCA, useHTTP)
 		if err != nil {
 			return errors.Wrap(err, "failed to create buildkit client")
 		}

diff --git a/pkg/app/prune.go b/pkg/app/prune.go
@@ -81,13 +81,13 @@ func prune(clicontext *cli.Context) error {
 	var bkClient buildkitd.Client
 	if c.Builder == types.BuilderTypeMoby {
 		bkClient, err = buildkitd.NewMobyClient(clicontext.Context,
-			c.Builder, c.BuilderAddress, "", false)
+			c.Builder, c.BuilderAddress, "", false, false)
 		if err != nil {
 			return errors.Wrap(err, "failed to create moby buildkit client")
 		}
 	} else {
 		bkClient, err = buildkitd.NewClient(clicontext.Context,
-			c.Builder, c.BuilderAddress, "", false)
+			c.Builder, c.BuilderAddress, "", false, false)
 		if err != nil {
 			return errors.Wrap(err, "failed to create buildkit client")
 		}

diff --git a/pkg/builder/build.go b/pkg/builder/build.go
@@ -90,13 +90,13 @@ func New(ctx context.Context, opt Options) (Builder, error) {
 	var cli buildkitd.Client
 	if c.Builder == types.BuilderTypeMoby {
 		cli, err = buildkitd.NewMobyClient(ctx,
-			c.Builder, c.BuilderAddress, "", false)
+			c.Builder, c.BuilderAddress, "", false, false)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create moby buildkit client")
 		}
 	} else {
 		cli, err = buildkitd.NewClient(ctx,
-			c.Builder, c.BuilderAddress, "", false)
+			c.Builder, c.BuilderAddress, "", false, false)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create buildkit client")
 		}

diff --git a/pkg/buildkitd/buildkitd.go b/pkg/buildkitd/buildkitd.go
@@ -66,6 +66,7 @@ type generalClient struct {
 	image            string
 	mirror           string
 	enableRegistryCA bool
+	useHTTP          bool
 
 	driver types.BuilderType
 	socket string
@@ -75,13 +76,14 @@ type generalClient struct {
 }
 
 func NewMobyClient(ctx context.Context, driver types.BuilderType,
-	socket, mirror string, enableRegistryCA bool) (Client, error) {
+	socket, mirror string, enableRegistryCA bool, useHTTP bool) (Client, error) {
 	logrus.Debug("getting moby buildkit client")
 	c := &generalClient{
 		containerName:    socket,
 		image:            viper.GetString(flag.FlagBuildkitdImage),
 		mirror:           mirror,
 		enableRegistryCA: enableRegistryCA,
+		useHTTP:          useHTTP,
 		socket:           socket,
 		driver:           driver,
 	}
@@ -104,19 +106,20 @@ func NewMobyClient(ctx context.Context, driver types.BuilderType,
 		}),
 	)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to create buildkit clientt")
+		return nil, errors.Wrap(err, "failed to create buildkit client")
 	}
 	c.Client = bkcli
 	return c, nil
 }
 
 func NewClient(ctx context.Context, driver types.BuilderType,
-	socket, mirror string, enableRegistryCA bool) (Client, error) {
+	socket, mirror string, enableRegistryCA bool, useHTTP bool) (Client, error) {
 	c := &generalClient{
 		containerName:    socket,
 		image:            viper.GetString(flag.FlagBuildkitdImage),
 		mirror:           mirror,
 		enableRegistryCA: enableRegistryCA,
+		useHTTP:          useHTTP,
 		socket:           socket,
 		driver:           driver,
 	}
@@ -174,7 +177,7 @@ func (c *generalClient) maybeStart(ctx context.Context,
 
 	if client != nil {
 		if _, err := client.StartBuildkitd(ctx, c.image, c.containerName, c.mirror,
-			c.enableRegistryCA, runningTimeout); err != nil {
+			c.enableRegistryCA, c.useHTTP, runningTimeout); err != nil {
 			return "", err
 		}
 	}

diff --git a/pkg/driver/client.go b/pkg/driver/client.go
@@ -25,7 +25,7 @@ import (
 type Client interface {
 	// Load loads the image from the reader to the docker host.
 	Load(ctx context.Context, r io.ReadCloser, quiet bool) error
-	StartBuildkitd(ctx context.Context, tag, name, mirror string, enableRegistryCA bool, timeout time.Duration) (string, error)
+	StartBuildkitd(ctx context.Context, tag, name, mirror string, enableRegistryCA, useHTTP bool, timeout time.Duration) (string, error)
 
 	Exec(ctx context.Context, cname string, cmd []string) error
 

diff --git a/pkg/driver/docker/docker.go b/pkg/driver/docker/docker.go
@@ -43,6 +43,10 @@ const buildkitdMirror = `
 [registry."docker.io"]
   mirrors = ["%s"]
 `
+const buildkitdHTTP = `
+[registry."docker.io"]
+  http = true
+`
 const buildkitdCertPath = "/etc/registry"
 const buildkitdRegistry = `
 [registry."docker.io"]
@@ -186,7 +190,7 @@ func (c dockerClient) ResumeContainer(ctx context.Context, name string) (string,
 }
 
 func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name, mirror string,
-	enableRegistryCA bool, timeout time.Duration) (string, error) {
+	enableRegistryCA, useHTTP bool, timeout time.Duration) (string, error) {
 	logger := logrus.WithFields(logrus.Fields{
 		"tag":       tag,
 		"container": name,
@@ -218,8 +222,8 @@ func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name, mirror stri
 		Privileged: true,
 		AutoRemove: true,
 	}
+	var cfg string
 	if mirror != "" {
-		var cfg string
 		if enableRegistryCA {
 			cfg = fmt.Sprintf(buildkitdRegistry, mirror)
 			hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{
@@ -230,13 +234,15 @@ func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name, mirror stri
 		} else {
 			cfg = fmt.Sprintf(buildkitdMirror, mirror)
 		}
-		config.Entrypoint = []string{
-			"/bin/sh",
-			"-c",
-			fmt.Sprintf("mkdir /etc/buildkit && echo '%s' > /etc/buildkit/buildkitd.toml && buildkitd", cfg),
-		}
-		logger.Debugf("setting buildkit config: %s", cfg)
+	} else if useHTTP {
+		cfg = buildkitdHTTP
+	}
+	config.Entrypoint = []string{
+		"/bin/sh",
+		"-c",
+		fmt.Sprintf("mkdir /etc/buildkit && echo '%s' > /etc/buildkit/buildkitd.toml && buildkitd", cfg),
 	}
+	logger.Debugf("setting buildkit config: %s", cfg)
 	created, _ := c.Exists(ctx, name)
 	if created {
 		err := c.ContainerStart(ctx, name, types.ContainerStartOptions{})

diff --git a/pkg/driver/nerdctl/nerdctl.go b/pkg/driver/nerdctl/nerdctl.go
@@ -64,7 +64,7 @@ func (nc *nerdctlClient) Load(ctx context.Context, r io.ReadCloser, quiet bool)
 }
 
 func (nc *nerdctlClient) StartBuildkitd(ctx context.Context, tag, name, mirror string,
-	enableRegistryCA bool, timeout time.Duration) (string, error) {
+	enableRegistryCA, useHTTP bool, timeout time.Duration) (string, error) {
 	logger := logrus.WithFields(logrus.Fields{
 		"tag":       tag,
 		"container": name,