Skip to content

Commit

Permalink
support using json for registry config (#1680)
Browse files Browse the repository at this point in the history 
* support using json for registry config

Signed-off-by: Richard Li <ricli@linkedin.com>

* simplify code

Signed-off-by: Richard Li <ricli@linkedin.com>

* Update pkg/app/bootstrap.go

Co-authored-by: Keming <kemingy94@gmail.com>
Signed-off-by: Richard Li <98242479+RichhLi@users.noreply.github.com>

---------

Signed-off-by: Richard Li <ricli@linkedin.com>
Signed-off-by: Richard Li <98242479+RichhLi@users.noreply.github.com>
Co-authored-by: Richard Li <ricli@linkedin.com>
Co-authored-by: Keming <kemingy94@gmail.com>
  • Loading branch information
@RichhLi @kemingy
3 people committed Jul 4, 2023
1 parent 5383e01 commit cde4632
Show file tree
Hide file tree
Showing 5 changed files with 217 additions and 41 deletions.
119 changes: 110 additions & 9 deletions pkg/app/bootstrap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package app

import (
"encoding/json"
"fmt"
"os"
"path/filepath"
Expand Down Expand Up @@ -75,6 +76,11 @@ var CommandBootstrap = &cli.Command{
Name: "registry",
Usage: "Specify the registry to pull the image from",
Aliases: []string{"r"},
Value: "docker.io",
},
&cli.StringFlag{
Name: "registry-config",
Usage: "Path to a JSON file containing registry configuration. Cannot be used with 'registry' or 'registry-ca-keypair'",
},
},

Expand All @@ -91,6 +97,9 @@ func bootstrap(clicontext *cli.Context) error {
}, {
"registry CA keypair",
registryCA,
}, {
"registry json config",
registryJSONConfig,
}, {
"autocomplete",
autocomplete,
Expand All @@ -112,16 +121,25 @@ func bootstrap(clicontext *cli.Context) error {
}

func registryCA(clicontext *cli.Context) error {
configFile := clicontext.String("registry-config")
ca := clicontext.String("registry-ca-keypair")
registry := clicontext.String("registry")

if len(ca) == 0 {
return nil
}

// We only need this check in registryCA because it is called before registryJSONConfig
if len(configFile) > 0 && len(ca) > 0 {
return errors.New("only one of `registry-config` and `registry-ca-keypair` can be used")
}

mirror := clicontext.String("dockerhub-mirror")
if len(mirror) == 0 {
return errors.New("`registry-ca-keypair` should be used with `dockerhub-mirror`")
}

// parse ca/key/cert
// Parse ca/key/cert
kvPairs := strings.Split(ca, ",")
if len(kvPairs) != 3 {
return errors.New("`registry-ca-keypair` requires ca/key/cert 3 part separated by ','")
Expand All @@ -146,7 +164,8 @@ func registryCA(clicontext *cli.Context) error {
if !exist {
return errors.Newf("file %s doesn't exist", kv[1])
}
path, err := fileutil.ConfigFile(fmt.Sprintf("registry_%s.pem", kv[0]))

path, err := fileutil.ConfigFile(fmt.Sprintf("%s_%s.pem", registry, kv[0]))
if err != nil {
return errors.Wrap(err, "failed to get the envd config file path")
}
Expand All @@ -166,6 +185,69 @@ func registryCA(clicontext *cli.Context) error {
return nil
}

func registryJSONConfig(clicontext *cli.Context) error {
configFile := clicontext.String("registry-config")
if len(configFile) == 0 {
return nil
}

// Check if file exists
exist, err := fileutil.FileExists(configFile)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to parse file path %s", configFile))
}
if !exist {
return errors.Newf("file %s doesn't exist", configFile)
}

config := buildkitutil.BuildkitConfig{}
configJson, err := os.ReadFile(configFile)
if err != nil {
return errors.Wrap(err, "Failed to read registry config file")
}
if err := json.Unmarshal(configJson, &config); err != nil {
return errors.Wrap(err, "Failed to parse registry config file")
}

// Check for required keys in each registry
for i, registry := range config.Registries {
if registry.Name == "" {
return errors.Newf("`name` key is required in the config for registry at index %d", i)
}

// Check for optional keys and if they exist, ensure they point to existing files
optionalKeys := map[string]string{"ca": registry.Ca, "cert": registry.Cert, "key": registry.Key}
for key, value := range optionalKeys {
if value != "" {
exist, err := fileutil.FileExists(value)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to parse file path %s", value))
}
if !exist {
return errors.Newf("file %s doesn't exist", value)
}

// Read the file
content, err := os.ReadFile(value)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to read the %s file for registry %s", key, registry.Name))
}

// Write the content to the envd config directory
envdConfigPath, err := fileutil.ConfigFile(fmt.Sprintf("%s_%s.pem", registry.Name, key))
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to get the envd config file path for %s of registry %s", key, registry.Name))
}

if err = os.WriteFile(envdConfigPath, content, 0644); err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to store the %s file for registry %s", key, registry.Name))
}
}
}
}
return nil
}

func sshKey(clicontext *cli.Context) error {
sshKeyPair := clicontext.StringSlice("ssh-keypair")

Expand Down Expand Up @@ -240,7 +322,7 @@ func sshKey(clicontext *cli.Context) error {
return nil

default:
return errors.Errorf("Invalid ssh-keypair flag")
return errors.Newf("Invalid ssh-keypair flag")
}
}

Expand Down Expand Up @@ -289,14 +371,33 @@ func buildkit(clicontext *cli.Context) error {
}

logrus.Debug("bootstrap the buildkitd container")
var bkClient buildkitd.Client
config := buildkitutil.BuildkitConfig{
Registry: clicontext.String("registry"),
Mirror: clicontext.String("dockerhub-mirror"),
UseHTTP: clicontext.Bool("use-http"),
SetCA: clicontext.IsSet("registry-ca-keypair"),
// Populate the BuildkitConfig struct
config := buildkitutil.BuildkitConfig{}

configFile := clicontext.String("registry-config")
if len(configFile) != 0 {
configJson, err := os.ReadFile(configFile)
if err != nil {
return errors.Wrap(err, "Failed to read registry config file")
}
if err := json.Unmarshal(configJson, &config); err != nil {
return errors.Wrap(err, "Failed to parse registry config file")
}
} else if len(clicontext.String("registry-ca-keypair")) != 0 {
// The values of Ca, Cert, and Key don't actually matter since we already copied their contents to the envd config directory and mounted to `/etc/registry`.
// So instead of parsing registry-ca-keypair again, we'll just put the default value.
// This is to ensure that buildkitConfigTemplate parses properly.
config.Registries = append(config.Registries, buildkitutil.Registry{
Name: clicontext.String("registry"),
Ca: "/etc/registry",
Cert: "/etc/registry",
Key: "/etc/registry",
UseHttp: clicontext.Bool("use-http"),
Mirror: clicontext.String("dockerhub-mirror"),
})
}

var bkClient buildkitd.Client
if c.Builder == types.BuilderTypeMoby {
bkClient, err = buildkitd.NewMobyClient(clicontext.Context,
c.Builder, c.BuilderAddress, &config)
Expand Down
3 changes: 2 additions & 1 deletion pkg/driver/docker/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,13 +207,14 @@ func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name string, bc *
AutoRemove: true,
}

if bc.SetCA {
if len(bc.Registries) > 0 {
hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{
Type: mount.TypeBind,
Source: fileutil.DefaultConfigDir,
Target: buildkitdCertPath,
})
}

cfg, err := bc.String()
if err != nil {
return "", errors.Wrap(err, "failed to generate buildkit config")
Expand Down
2 changes: 1 addition & 1 deletion pkg/driver/nerdctl/nerdctl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func (nc *nerdctlClient) StartBuildkitd(ctx context.Context, tag, name string, b
if !existed {
buildkitdCmd := "buildkitd"
// TODO: support mirror CA keypair
if bc.Registry != "" || bc.Mirror != "" || bc.UseHTTP {
if len(bc.Registries) > 0 {
cfg, err := bc.String()
if err != nil {
return "", errors.Wrap(err, "failed to generate buildkit config")
Expand Down
40 changes: 28 additions & 12 deletions pkg/util/buildkitutil/buildkit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,37 @@ import (
)

const buildkitConfigTemplate = `
[registry."{{ if .Registry }}{{ .Registry }}{{ else }}docker.io{{ end }}"]{{ if .Mirror }}
mirrors = ["{{ .Mirror }}"]{{ end }}
http = {{ .UseHTTP }}
{{ if .SetCA}}ca=["/etc/registry/ca.pem"]
[[registry."{{ if .Registry }}{{ .Registry }}{{ else }}docker.io{{ end }}".keypair]]
key="/etc/registry/key.pem"
cert="/etc/registry/cert.pem"
{{ end }}
[registry]
{{- range $registry := .Registries }}
[registry."{{ if $registry.Name }}{{ $registry.Name }}{{ else }}docker.io{{ end }}"]
{{- if $registry.UseHttp }}
http = true
{{- end }}
{{- if $registry.Mirror }}
mirrors = ["{{ $registry.Mirror }}"]
{{- end }}
{{- if $registry.Ca }}
ca=["/etc/registry/{{ $registry.Name }}_ca.pem"]
{{- end }}
{{- if and $registry.Cert $registry.Key }}
[[registry."{{ if $registry.Name }}{{ $registry.Name }}{{ else }}docker.io{{ end }}".keypair]]
key="/etc/registry/{{ $registry.Name }}_key.pem"
cert="/etc/registry/{{ $registry.Name }}_cert.pem"
{{- end }}
{{- end }}
`

type Registry struct {
Name string `json:"name"`
Ca string `json:"ca"`
Cert string `json:"cert"`
Key string `json:"key"`
UseHttp bool `json:"use_http"`
Mirror string `json:"mirror"`
}

type BuildkitConfig struct {
Registry string
Mirror string
UseHTTP bool
SetCA bool
Registries []Registry `json:"registries"`
}

func (c *BuildkitConfig) String() (string, error) {
Expand Down
94 changes: 76 additions & 18 deletions pkg/util/buildkitutil/buildkit_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,36 +28,94 @@ func TestBuildkitWithRegistry(t *testing.T) {
}{
{
BuildkitConfig{
Registry: "registry.example.com",
Mirror: "https://mirror.example.com",
UseHTTP: true,
Registries: []Registry{
{
Name: "registry.example.com",
Ca: "/etc/registry/ca.pem",
Cert: "/etc/registry/cert.pem",
Key: "/etc/registry/key.pem",
UseHttp: false,
Mirror: "https://mirror.example.com",
},
},
},
`
[registry."registry.example.com"]
mirrors = ["https://mirror.example.com"]
http = true
[registry]
[registry."registry.example.com"]
mirrors = ["https://mirror.example.com"]
ca=["/etc/registry/registry.example.com_ca.pem"]
[[registry."registry.example.com".keypair]]
key="/etc/registry/registry.example.com_key.pem"
cert="/etc/registry/registry.example.com_cert.pem"
`,
},
{
BuildkitConfig{
Registry: "registry.example.com",
SetCA: true,
Registries: []Registry{
{
Name: "registry.example.com",
UseHttp: true,
Mirror: "https://mirror.example.com",
},
{
Name: "docker.io",
Mirror: "https://mirror.example.com",
},
},
},
`
[registry."registry.example.com"]
http = false
ca=["/etc/registry/ca.pem"]
[[registry."registry.example.com".keypair]]
key="/etc/registry/key.pem"
cert="/etc/registry/cert.pem"
[registry]
[registry."registry.example.com"]
http = true
mirrors = ["https://mirror.example.com"]
[registry."docker.io"]
mirrors = ["https://mirror.example.com"]
`,
},
{
BuildkitConfig{},
BuildkitConfig{
Registries: []Registry{},
},
`
[registry."docker.io"]
http = false
`,
[registry]
`,
},
{
BuildkitConfig{
Registries: []Registry{
{
Name: "registry1.example.com",
Ca: "/etc/registry/ca1.pem",
Cert: "/etc/registry/cert1.pem",
Key: "/etc/registry/key1.pem",
UseHttp: true,
Mirror: "https://mirror.example.com",
},
{
Name: "registry2.example.com",
Ca: "/etc/registry/ca2.pem",
Cert: "/etc/registry/cert2.pem",
Key: "/etc/registry/key2.pem",
Mirror: "https://mirror.example.com",
},
},
},
`
[registry]
[registry."registry1.example.com"]
http = true
mirrors = ["https://mirror.example.com"]
ca=["/etc/registry/registry1.example.com_ca.pem"]
[[registry."registry1.example.com".keypair]]
key="/etc/registry/registry1.example.com_key.pem"
cert="/etc/registry/registry1.example.com_cert.pem"
[registry."registry2.example.com"]
mirrors = ["https://mirror.example.com"]
ca=["/etc/registry/registry2.example.com_ca.pem"]
[[registry."registry2.example.com".keypair]]
key="/etc/registry/registry2.example.com_key.pem"
cert="/etc/registry/registry2.example.com_cert.pem"
`,
},
}

Expand Down

0 comments on commit cde4632

Please sign in to comment.