Skip to content

Commit

Permalink
feat(CLI): Add push (#531)
Browse files Browse the repository at this point in the history 
* feat(CLI): Support build-and-push

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* fix: Update

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* fix: Remove unused test

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* fix: Remove unused test

Signed-off-by: Ce Gao <cegao@tensorchord.ai>
  • Loading branch information
@gaocegege
gaocegege committed Jul 1, 2022
1 parent 956fd73 commit 8276d7d
Show file tree
Hide file tree
Showing 9 changed files with 252 additions and 132 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,5 @@ share/python-wheels/
.installed.cfg
*.egg
MANIFEST

test.tar
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v20.10.13+incompatible // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/getsentry/sentry-go v0.12.0 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand All @@ -150,6 +151,8 @@ github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6
github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v20.10.3-0.20220224222438-c78f6963a1c0+incompatible h1:Ptj2To+ezU/mCBUKdYXBQ2r3/2EJojAlOZrsgprF+is=
github.com/docker/docker v20.10.3-0.20220224222438-c78f6963a1c0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=
Expand Down
23 changes: 16 additions & 7 deletions pkg/app/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ var CommandBuild = &cli.Command{
Name: "build",
Aliases: []string{"b"},
Usage: "Build the envd environment",
Description: `
To build an image using build.envd:
$ envd build
To build and push the image to a registry:
$ envd build --output type=image,name=docker.io/username/image,push=true
`,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "tag",
Expand All @@ -59,11 +65,10 @@ var CommandBuild = &cli.Command{
Value: sshconfig.GetPublicKey(),
Hidden: true,
},
&cli.PathFlag{
&cli.StringFlag{
Name: "output",
Usage: "Output destination (format: type=tar,dest=path)",
Usage: "Output destination (e.g. type=tar,dest=path,push=true)",
Aliases: []string{"o"},
Value: "",
},
},

Expand All @@ -84,7 +89,7 @@ func build(clicontext *cli.Context) error {
return errors.New("file does not exist")
}

config := home.GetManager().ConfigFile()
cfg := home.GetManager().ConfigFile()

tag := clicontext.String("tag")
if tag == "" {
Expand All @@ -95,13 +100,17 @@ func build(clicontext *cli.Context) error {
logger := logrus.WithFields(logrus.Fields{
"build-context": buildContext,
"build-file": manifest,
"config": config,
"config": cfg,
"tag": tag,
flag.FlagBuildkitdImage: viper.GetString(flag.FlagBuildkitdImage),
})
logger.Debug("starting build command")
debug := clicontext.Bool("debug")
builder, err := builder.New(clicontext.Context, config, manifest, buildContext, tag, clicontext.Path("output"), debug)
output := clicontext.String("output")
logger.WithFields(logrus.Fields{
"output": output,
}).Debug("starting build command")
builder, err := builder.New(clicontext.Context, cfg,
manifest, buildContext, tag, output, debug)
if err != nil {
return errors.Wrap(err, "failed to create the builder")
}
Expand Down
4 changes: 3 additions & 1 deletion pkg/app/up.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,9 @@ func up(clicontext *cli.Context) error {
})
logger.Debug("starting up command")
debug := clicontext.Bool("debug")
builder, err := builder.New(clicontext.Context, config, manifest, buildContext, tag, "", debug)
output := ""
builder, err := builder.New(clicontext.Context, config, manifest,
buildContext, tag, output, debug)
if err != nil {
return errors.Wrap(err, "failed to create the builder")
}
Expand Down
181 changes: 103 additions & 78 deletions pkg/builder/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
"github.com/cockroachdb/errors"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/auth/authprovider"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"

Expand All @@ -48,19 +50,33 @@ type generalBuilder struct {
progressMode string
tag string
buildContextDir string
outputType string
outputDest string

entries []client.ExportEntry

logger *logrus.Entry
starlark.Interpreter
buildkitd.Client
}

func New(ctx context.Context, configFilePath, manifestFilePath, buildContextDir, tag, output string, debug bool) (Builder, error) {
outputType, outputDest, err := parseOutput(output)
func New(ctx context.Context, configFilePath, manifestFilePath,
buildContextDir, tag string, output string, debug bool) (Builder, error) {
entries, err := parseOutput(output)
if err != nil {
return nil, errors.Wrap(err, "failed to parse output")
}

logrus.WithField("entry", entries).Debug("getting exporter entry")
// Build docker image by default
if len(entries) == 0 {
entries = []client.ExportEntry{
{
Type: client.ExporterDocker,
},
}
} else if len(entries) > 1 {
return nil, errors.New("only one output type is supported")
}

var mode string = "auto"
if debug {
mode = "plain"
Expand All @@ -69,12 +85,10 @@ func New(ctx context.Context, configFilePath, manifestFilePath, buildContextDir,
b := &generalBuilder{
manifestFilePath: manifestFilePath,
configFilePath: configFilePath,
outputType: outputType,
outputDest: outputDest,
buildContextDir: buildContextDir,
// TODO(gaocegege): Support other mode?
progressMode: mode,
tag: tag,
entries: entries,
progressMode: mode,
tag: tag,
logger: logrus.WithFields(logrus.Fields{
"tag": tag,
}),
Expand Down Expand Up @@ -171,39 +185,89 @@ func (b generalBuilder) build(ctx context.Context, def *llb.Definition, pw progr
// Create a pipe to load the image into the docker host.
pipeR, pipeW := io.Pipe()

eg.Go(func() error {
defer pipeW.Close()
_, err := b.Solve(ctx, def, client.SolveOpt{
Exports: []client.ExportEntry{
{
Type: client.ExporterDocker,
Attrs: map[string]string{
"name": b.tag,
// Ref https://github.com/r2d4/mockerfile/blob/140c6a912bbfdae220febe59ab535ef0acba0e1f/pkg/build/build.go#L65
"containerimage.config": labels,
for _, entry := range b.entries {
// Set up docker config auth.
attachable := []session.Attachable{authprovider.NewDockerAuthProvider(os.Stderr)}
switch entry.Type {
// Create default build.
case client.ExporterDocker:
eg.Go(func() error {
if entry.Attrs == nil {
entry = client.ExportEntry{
Type: client.ExporterDocker,
Attrs: map[string]string{
"name": b.tag,
// Ref https://github.com/r2d4/mockerfile/blob/140c6a912bbfdae220febe59ab535ef0acba0e1f/pkg/build/build.go#L65
"containerimage.config": labels,
},
Output: func(map[string]string) (io.WriteCloser, error) {
return pipeW, nil
},
}
}
defer pipeW.Close()
_, err := b.Solve(ctx, def, client.SolveOpt{
Exports: []client.ExportEntry{entry},
LocalDirs: map[string]string{
flag.FlagContextDir: b.buildContextDir,
flag.FlagCacheDir: home.GetManager().CacheDir(),
},
Output: func(map[string]string) (io.WriteCloser, error) {
return pipeW, nil
Session: attachable,
// TODO(gaocegege): Use llb.WithProxy to implement it.
FrontendAttrs: map[string]string{
"build-arg:HTTPS_PROXY": os.Getenv("HTTPS_PROXY"),
},
},
},
LocalDirs: map[string]string{
flag.FlagContextDir: b.buildContextDir,
flag.FlagCacheDir: home.GetManager().CacheDir(),
},
// TODO(gaocegege): Use llb.WithProxy to implement it.
FrontendAttrs: map[string]string{
"build-arg:HTTPS_PROXY": os.Getenv("HTTPS_PROXY"),
},
}, pw.Status())
if err != nil {
err = errors.Wrap(err, "failed to solve LLB")
b.logger.Error(err)
return err
}, pw.Status())

if err != nil {
err = errors.Wrap(err, "failed to solve LLB")
b.logger.Error(err)
return err
}
b.logger.Debug("llb def is solved successfully")
return nil
})
// Load the image to docker host.
eg.Go(func() error {
defer pipeR.Close()
dockerClient, err := docker.NewClient(ctx)
if err != nil {
return errors.Wrap(err, "failed to new docker client")
}
b.logger.Debug("loading image to docker host")
if err := dockerClient.Load(ctx, pipeR, true); err != nil {
err = errors.Wrap(err, "failed to load docker image")
b.logger.Error(err)
return err
}
b.logger.Debug("loaded docker image successfully")
return nil
})
default:
eg.Go(func() error {
_, err := b.Solve(ctx, def, client.SolveOpt{
Exports: []client.ExportEntry{entry},
LocalDirs: map[string]string{
flag.FlagContextDir: b.buildContextDir,
flag.FlagCacheDir: home.GetManager().CacheDir(),
},
Session: attachable,
// TODO(gaocegege): Use llb.WithProxy to implement it.
FrontendAttrs: map[string]string{
"build-arg:HTTPS_PROXY": os.Getenv("HTTPS_PROXY"),
},
}, pw.Status())

if err != nil {
err = errors.Wrap(err, "failed to solve LLB")
b.logger.Error(err)
return err
}
b.logger.Debug("llb def is solved successfully")
return nil
})
}
b.logger.Debug("llb def is solved successfully")
return nil
})
}

// Watch the progress.
eg.Go(func() error {
Expand All @@ -212,45 +276,6 @@ func (b generalBuilder) build(ctx context.Context, def *llb.Definition, pw progr
return pw.Err()
})

if b.outputDest != "" {
// Save the image to the output file.
eg.Go(func() error {
defer pipeR.Close()
f, err := os.Create(b.outputDest)
if err != nil {
return err
}

defer f.Close()
_, err = io.Copy(f, pipeR)
if err != nil {
return err
}

b.logger.Debug("export the image successfully")
return nil
})
}

if b.outputDest == "" {
// Load the image to docker host.
eg.Go(func() error {
defer pipeR.Close()
dockerClient, err := docker.NewClient(ctx)
if err != nil {
return errors.Wrap(err, "failed to new docker client")
}
b.logger.Debug("loading image to docker host")
if err := dockerClient.Load(ctx, pipeR, true); err != nil {
err = errors.Wrap(err, "failed to load docker image")
b.logger.Error(err)
return err
}
b.logger.Debug("loaded docker image successfully")
return nil
})
}

err = eg.Wait()
if err != nil {
if errors.Is(err, context.Canceled) {
Expand Down
9 changes: 1 addition & 8 deletions pkg/builder/builder_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,13 @@ import (

var _ = Describe("Builder", func() {
Describe("building image", Label("buildkitd"), func() {
var configFilePath, manifestFilePath, buildContext, tag string
var configFilePath, manifestFilePath, tag string
BeforeEach(func() {
configFilePath = "config.envd"
manifestFilePath = "build.envd"
buildContext = "testdata"
tag = "envd-dev:test"
Expect(home.Initialize()).NotTo(HaveOccurred())
})
When("getting the wrong builtkitd address", func() {
It("should return an error", func() {
_, err := New(context.TODO(), configFilePath, manifestFilePath, buildContext, tag, "", false)
Expect(err).To(HaveOccurred())
})
})
When("building the manifest", func() {
var b *generalBuilder
var w compileui.Writer
Expand Down
Loading

0 comments on commit 8276d7d

Please sign in to comment.