From ea859289fde98aca013b05488209b5590bcefa5a Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 17 Jun 2016 07:04:08 -0700 Subject: [PATCH] image: Refactor to use cas/ref engines instead of walkers The validation/unpacking code doesn't really care what the reference and CAS implemenations are. And the new generic interfaces in image/refs and image/cas will scale better as we add new backends than the walker interface. The old tar/directory distinction between image and imageLayout is gone. The new CAS/refs engines don't support directory backends yet (I plan on adding them once the engine framework lands), but the new framework will handle tar/directory/... detection inside layout.NewEngine (and possibly inside a new (cas|refs).NewEngine when we grow engine types that aren't based on image-layout). Also replace the old methods like: func (d *descriptor) validateContent(r io.Reader) error with functions like: validateContent(ctx context.Context, descriptor *specs.Descriptor, r io.Reader) error to avoid local types that duplicate the image-spec types. This saves an extra instantiation for folks who want to validate (or whatever) a specs.Descriptor they have obtained elsewhere. I'd prefer casLayout and refsLayout for the imported packages, but Stephen doesn't want camelCase for package names [1]. [1]: https://github.com/opencontainers/image-spec/pull/159#discussion_r76720225 Signed-off-by: W. Trevor King --- cmd/oci-create-runtime-bundle/main.go | 9 +- .../oci-create-runtime-bundle.1.md | 2 +- cmd/oci-image-validate/main.go | 14 +- .../oci-image-validate.1.md | 6 +- cmd/oci-unpack/main.go | 13 +- cmd/oci-unpack/oci-unpack.1.md | 2 +- image/autodetect.go | 3 +- image/config.go | 69 +++--- image/descriptor.go | 115 ++-------- image/image.go | 202 ++++++++++-------- image/manifest.go | 119 +++++------ image/manifest_test.go | 45 ++-- image/walker.go | 118 ---------- 13 files changed, 263 insertions(+), 454 deletions(-) delete mode 100644 image/walker.go diff --git a/cmd/oci-create-runtime-bundle/main.go b/cmd/oci-create-runtime-bundle/main.go index e9b99ab..a56bcee 100644 --- a/cmd/oci-create-runtime-bundle/main.go +++ b/cmd/oci-create-runtime-bundle/main.go @@ -22,11 +22,11 @@ import ( "github.com/opencontainers/image-tools/image" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // supported bundle types var bundleTypes = []string{ - image.TypeImageLayout, image.TypeImage, } @@ -93,6 +93,8 @@ func (v *bundleCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + if _, err := os.Stat(args[1]); os.IsNotExist(err) { v.stderr.Printf("destination path %s does not exist", args[1]) os.Exit(1) @@ -109,11 +111,8 @@ func (v *bundleCmd) Run(cmd *cobra.Command, args []string) { var err error switch v.typ { - case image.TypeImageLayout: - err = image.CreateRuntimeBundleLayout(args[0], args[1], v.ref, v.root) - case image.TypeImage: - err = image.CreateRuntimeBundle(args[0], args[1], v.ref, v.root) + err = image.CreateRuntimeBundle(ctx, args[0], args[1], v.ref, v.root) } if err != nil { diff --git a/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md b/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md index ad95181..8fee487 100644 --- a/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md +++ b/cmd/oci-create-runtime-bundle/oci-create-runtime-bundle.1.md @@ -24,7 +24,7 @@ runtime-spec-compatible `dest/config.json`. A directory representing the root filesystem of the container in the OCI runtime bundle. It is strongly recommended to keep the default value. (default "rootfs") **--type** - Type of the file to unpack. If unset, oci-create-runtime-bundle will try to auto-detect the type. One of "imageLayout,image" + Type of the file to unpack. If unset, oci-create-runtime-bundle will try to auto-detect the type. One of "image" # EXAMPLES ``` diff --git a/cmd/oci-image-validate/main.go b/cmd/oci-image-validate/main.go index f8849bd..ca8e608 100644 --- a/cmd/oci-image-validate/main.go +++ b/cmd/oci-image-validate/main.go @@ -24,11 +24,11 @@ import ( "github.com/opencontainers/image-tools/image" "github.com/pkg/errors" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // supported validation types var validateTypes = []string{ - image.TypeImageLayout, image.TypeImage, image.TypeManifest, image.TypeManifestList, @@ -75,7 +75,7 @@ func newValidateCmd(stdout, stderr *log.Logger) *cobra.Command { cmd.Flags().StringSliceVar( &v.refs, "ref", nil, - `A set of refs pointing to the manifests to be validated. Each reference must be present in the "refs" subdirectory of the image. Only applicable if type is image or imageLayout.`, + `A set of refs pointing to the manifests to be validated. Each reference must be present in the "refs" subdirectory of the image. Only applicable if type is image.`, ) return cmd @@ -90,9 +90,11 @@ func (v *validateCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + var exitcode int for _, arg := range args { - err := v.validatePath(arg) + err := v.validatePath(ctx, arg) if err == nil { v.stdout.Printf("%s: OK", arg) @@ -122,7 +124,7 @@ func (v *validateCmd) Run(cmd *cobra.Command, args []string) { os.Exit(exitcode) } -func (v *validateCmd) validatePath(name string) error { +func (v *validateCmd) validatePath(ctx context.Context, name string) error { var ( err error typ = v.typ @@ -135,10 +137,8 @@ func (v *validateCmd) validatePath(name string) error { } switch typ { - case image.TypeImageLayout: - return image.ValidateLayout(name, v.refs, v.stdout) case image.TypeImage: - return image.Validate(name, v.refs, v.stdout) + return image.Validate(ctx, name, v.refs, v.stdout) } f, err := os.Open(name) diff --git a/cmd/oci-image-validate/oci-image-validate.1.md b/cmd/oci-image-validate/oci-image-validate.1.md index 33d2b49..cf04249 100644 --- a/cmd/oci-image-validate/oci-image-validate.1.md +++ b/cmd/oci-image-validate/oci-image-validate.1.md @@ -20,15 +20,15 @@ oci-image-validate \- Validate one or more image files Can be specified multiple times to validate multiple references. `NAME` must be present in the `refs` subdirectory of the image. Defaults to `v1.0`. - Only applicable if type is image or imageLayout. + Only applicable if type is image. **--type** - Type of the file to validate. If unset, oci-image-validate will try to auto-detect the type. One of "imageLayout,image,manifest,manifestList,config" + Type of the file to validate. If unset, oci-image-validate will try to auto-detect the type. One of "image,manifest,manifestList,config" # EXAMPLES ``` $ skopeo copy docker://busybox oci:busybox-oci -$ oci-image-validate --type imageLayout --ref latest busybox-oci +$ oci-image-validate --type image --ref latest busybox-oci busybox-oci: OK ``` diff --git a/cmd/oci-unpack/main.go b/cmd/oci-unpack/main.go index 463db62..3944dcc 100644 --- a/cmd/oci-unpack/main.go +++ b/cmd/oci-unpack/main.go @@ -22,11 +22,11 @@ import ( "github.com/opencontainers/image-tools/image" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // supported unpack types var unpackTypes = []string{ - image.TypeImageLayout, image.TypeImage, } @@ -56,8 +56,8 @@ func newUnpackCmd(stdout, stderr *log.Logger) *cobra.Command { cmd := &cobra.Command{ Use: "unpack [src] [dest]", - Short: "Unpack an image or image source layout", - Long: `Unpack the OCI image .tar file or OCI image layout directory present at [src] to the destination directory [dest].`, + Short: "Unpack an image", + Long: `Unpack the OCI image present at [src] to the destination directory [dest].`, Run: v.Run, } @@ -86,6 +86,8 @@ func (v *unpackCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + if v.typ == "" { typ, err := image.Autodetect(args[0]) if err != nil { @@ -97,11 +99,8 @@ func (v *unpackCmd) Run(cmd *cobra.Command, args []string) { var err error switch v.typ { - case image.TypeImageLayout: - err = image.UnpackLayout(args[0], args[1], v.ref) - case image.TypeImage: - err = image.Unpack(args[0], args[1], v.ref) + err = image.Unpack(ctx, args[0], args[1], v.ref) } if err != nil { diff --git a/cmd/oci-unpack/oci-unpack.1.md b/cmd/oci-unpack/oci-unpack.1.md index 890bd0f..9883c51 100644 --- a/cmd/oci-unpack/oci-unpack.1.md +++ b/cmd/oci-unpack/oci-unpack.1.md @@ -18,7 +18,7 @@ oci-unpack \- Unpack an image or image source layout The ref pointing to the manifest to be unpacked. This must be present in the "refs" subdirectory of the image. (default "v1.0") **--type** - Type of the file to unpack. If unset, oci-unpack will try to auto-detect the type. One of "imageLayout,image" + Type of the file to unpack. If unset, oci-unpack will try to auto-detect the type. One of "image" # EXAMPLES ``` diff --git a/image/autodetect.go b/image/autodetect.go index c3e2f85..5b0552e 100644 --- a/image/autodetect.go +++ b/image/autodetect.go @@ -27,7 +27,6 @@ import ( // supported autodetection types const ( - TypeImageLayout = "imageLayout" TypeImage = "image" TypeManifest = "manifest" TypeManifestList = "manifestList" @@ -43,7 +42,7 @@ func Autodetect(path string) (string, error) { } if fi.IsDir() { - return TypeImageLayout, nil + return TypeImage, nil } f, err := os.Open(path) diff --git a/image/config.go b/image/config.go index 14e41e2..11a90c1 100644 --- a/image/config.go +++ b/image/config.go @@ -18,58 +18,57 @@ import ( "bytes" "encoding/json" "fmt" - "io" "io/ioutil" - "os" - "path/filepath" "strconv" "strings" "github.com/opencontainers/image-spec/schema" + imagespecs "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/image-tools/image/cas" + runtimespecs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" + "golang.org/x/net/context" ) -type config v1.Image +func findConfig(ctx context.Context, engine cas.Engine, descriptor *imagespecs.Descriptor) (config *v1.Image, err error) { + err = validateMediaType(descriptor.MediaType, []string{v1.MediaTypeImageConfig}) + if err != nil { + return nil, errors.Wrap(err, "invalid config media type") + } -func findConfig(w walker, d *descriptor) (*config, error) { - var c config - cpath := filepath.Join("blobs", d.algo(), d.hash()) + if err := validateDescriptor(ctx, engine, descriptor); err != nil { + return nil, errors.Wrap(err, "invalid config descriptor") + } - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != cpath { - return nil - } - buf, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrapf(err, "%s: error reading config", path) - } + reader, err := engine.Get(ctx, descriptor.Digest) + if err != nil { + return nil, errors.Wrapf(err, "failed to fetch %s", descriptor.Digest) + } - if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil { - return errors.Wrapf(err, "%s: config validation failed", path) - } + buf, err := ioutil.ReadAll(reader) + if err != nil { + return nil, errors.Wrapf(err, "%s: error reading manifest", descriptor.Digest) + } - if err := json.Unmarshal(buf, &c); err != nil { - return err - } - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: config not found", cpath) - case errEOW: - return &c, nil - default: + if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil { + return nil, errors.Wrapf(err, "%s: config validation failed", descriptor.Digest) + } + + var c v1.Image + if err := json.Unmarshal(buf, &c); err != nil { return nil, err } + + return &c, nil } -func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { +func runtimeSpec(c *v1.Image, rootfs string) (*runtimespecs.Spec, error) { if c.OS != "linux" { return nil, fmt.Errorf("%s: unsupported OS", c.OS) } - var s specs.Spec + var s runtimespecs.Spec s.Version = "0.5.0" // we should at least apply the default spec, otherwise this is totally useless s.Process.Terminal = true @@ -112,12 +111,12 @@ func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { swap := uint64(c.Config.MemorySwap) shares := uint64(c.Config.CPUShares) - s.Linux.Resources = &specs.Resources{ - CPU: &specs.CPU{ + s.Linux.Resources = &runtimespecs.Resources{ + CPU: &runtimespecs.CPU{ Shares: &shares, }, - Memory: &specs.Memory{ + Memory: &runtimespecs.Memory{ Limit: &mem, Reservation: &mem, Swap: &swap, @@ -127,7 +126,7 @@ func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { for vol := range c.Config.Volumes { s.Mounts = append( s.Mounts, - specs.Mount{ + runtimespecs.Mount{ Destination: vol, Type: "bind", Options: []string{"rbind"}, diff --git a/image/descriptor.go b/image/descriptor.go index 341b65c..73eaf89 100644 --- a/image/descriptor.go +++ b/image/descriptor.go @@ -17,119 +17,34 @@ package image import ( "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" - "os" - "path/filepath" - "strings" + "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-tools/image/cas" "github.com/pkg/errors" + "golang.org/x/net/context" ) -type descriptor struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size int64 `json:"size"` -} - -func (d *descriptor) algo() string { - pts := strings.SplitN(d.Digest, ":", 2) - if len(pts) != 2 { - return "" - } - return pts[0] -} - -func (d *descriptor) hash() string { - pts := strings.SplitN(d.Digest, ":", 2) - if len(pts) != 2 { - return "" - } - return pts[1] -} - -func listReferences(w walker) (map[string]*descriptor, error) { - refs := make(map[string]*descriptor) - - if err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || !strings.HasPrefix(path, "refs") { +func validateMediaType(mediaType string, mediaTypes []string) error { + for _, mt := range mediaTypes { + if mt == mediaType { return nil } - - var d descriptor - if err := json.NewDecoder(r).Decode(&d); err != nil { - return err - } - refs[info.Name()] = &d - - return nil - }); err != nil { - return nil, err } - return refs, nil + return fmt.Errorf("invalid media type %q", mediaType) } -func findDescriptor(w walker, name string) (*descriptor, error) { - var d descriptor - dpath := filepath.Join("refs", name) - - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != dpath { - return nil - } - - if err := json.NewDecoder(r).Decode(&d); err != nil { - return err - } - - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: descriptor not found", dpath) - case errEOW: - return &d, nil - default: - return nil, err +func validateDescriptor(ctx context.Context, engine cas.Engine, descriptor *specs.Descriptor) error { + reader, err := engine.Get(ctx, descriptor.Digest) + if err != nil { + return errors.Wrapf(err, "failed to fetch %s", descriptor.Digest) } -} -func (d *descriptor) validate(w walker, mts []string) error { - var found bool - for _, mt := range mts { - if d.MediaType == mt { - found = true - break - } - } - if !found { - return fmt.Errorf("invalid descriptor MediaType %q", d.MediaType) - } - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() { - return nil - } - - filename, err := filepath.Rel(filepath.Join("blobs", d.algo()), filepath.Clean(path)) - if err != nil || d.hash() != filename { - return nil - } - - if err := d.validateContent(r); err != nil { - return err - } - return errEOW - }); err { - case nil: - return fmt.Errorf("%s: not found", d.Digest) - case errEOW: - return nil - default: - return errors.Wrapf(err, "%s: validation failed", d.Digest) - } + return validateContent(ctx, descriptor, reader) } -func (d *descriptor) validateContent(r io.Reader) error { +func validateContent(ctx context.Context, descriptor *specs.Descriptor, r io.Reader) error { h := sha256.New() n, err := io.Copy(h, r) if err != nil { @@ -138,11 +53,11 @@ func (d *descriptor) validateContent(r io.Reader) error { digest := "sha256:" + hex.EncodeToString(h.Sum(nil)) - if digest != d.Digest { + if digest != descriptor.Digest { return errors.New("digest mismatch") } - if n != d.Size { + if n != descriptor.Size { return errors.New("size mismatch") } diff --git a/image/image.go b/image/image.go index 095f48f..55c7267 100644 --- a/image/image.go +++ b/image/image.go @@ -16,176 +16,200 @@ package image import ( "encoding/json" - "fmt" "log" "os" "path/filepath" "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/image-tools/image/cas" + caslayout "github.com/opencontainers/image-tools/image/cas/layout" + "github.com/opencontainers/image-tools/image/refs" + refslayout "github.com/opencontainers/image-tools/image/refs/layout" "github.com/pkg/errors" + "golang.org/x/net/context" ) -// ValidateLayout walks through the given file tree and validates the manifest -// pointed to by the given refs or returns an error if the validation failed. -func ValidateLayout(src string, refs []string, out *log.Logger) error { - return validate(newPathWalker(src), refs, out) -} - -// Validate walks through the given .tar file and validates the manifest -// pointed to by the given refs or returns an error if the validation failed. -func Validate(tarFile string, refs []string, out *log.Logger) error { - f, err := os.Open(tarFile) - if err != nil { - return errors.Wrap(err, "unable to open file") - } - defer f.Close() - - return validate(newTarWalker(f), refs, out) -} - var validRefMediaTypes = []string{ v1.MediaTypeImageManifest, v1.MediaTypeImageManifestList, } -func validate(w walker, refs []string, out *log.Logger) error { - ds, err := listReferences(w) +// Validate validates the given reference. +func Validate(ctx context.Context, path string, refs []string, out *log.Logger) error { + refEngine, err := refslayout.NewEngine(ctx, path) if err != nil { return err } - if len(refs) == 0 && len(ds) == 0 { + defer refEngine.Close() + + casEngine, err := caslayout.NewEngine(ctx, path) + if err != nil { + return err + } + defer casEngine.Close() + + if len(refs) > 0 { + for _, ref := range refs { + err = validate(ctx, refEngine, casEngine, ref, out) + if err != nil { + return err + } + } + } + + count := 0 + err = refEngine.List( + ctx, + "", + -1, + 0, + func(ctx context.Context, name string) error { + count++ + return validate(ctx, refEngine, casEngine, name, out) + }, + ) + + if count == 0 { // TODO(runcom): ugly, we'll need a better way and library // to express log levels. // see https://github.com/opencontainers/image-spec/issues/288 out.Print("WARNING: no descriptors found") } - if len(refs) == 0 { - for ref := range ds { - refs = append(refs, ref) - } + return nil +} + +func validate(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, ref string, out *log.Logger) error { + descriptor, err := refEngine.Get(ctx, ref) + if err != nil { + return errors.Wrapf(err, "failed to fetch %q", ref) } - for _, ref := range refs { - d, ok := ds[ref] - if !ok { - // TODO(runcom): - // soften this error to a warning if the user didn't ask for any specific reference - // with --ref but she's just validating the whole image. - return fmt.Errorf("reference %s not found", ref) - } + err = validateMediaType(descriptor.MediaType, validRefMediaTypes) + if err != nil { + return err + } - if err = d.validate(w, validRefMediaTypes); err != nil { - return err - } + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { + return err + } - m, err := findManifest(w, d) - if err != nil { - return err - } + m, err := findManifest(ctx, casEngine, descriptor) + if err != nil { + return err + } - if err := m.validate(w); err != nil { - return err - } - if out != nil { - out.Printf("reference %q: OK", ref) - } + err = validateManifest(ctx, m, casEngine) + if err != nil { + return err + } + + if out != nil { + out.Printf("reference %q: OK", ref) } return nil } -// UnpackLayout walks through the file tree given by src and, using the layers -// specified in the manifest pointed to by the given ref, unpacks all layers in -// the given destination directory or returns an error if the unpacking failed. -func UnpackLayout(src, dest, ref string) error { - return unpack(newPathWalker(src), dest, ref) -} +// Unpack unpacks the given reference to a destination directory. +func Unpack(ctx context.Context, path, dest, ref string) error { + refEngine, err := refslayout.NewEngine(ctx, path) + if err != nil { + return err + } + defer refEngine.Close() -// Unpack walks through the given .tar file and, using the layers specified in -// the manifest pointed to by the given ref, unpacks all layers in the given -// destination directory or returns an error if the unpacking failed. -func Unpack(tarFile, dest, ref string) error { - f, err := os.Open(tarFile) + casEngine, err := caslayout.NewEngine(ctx, path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer casEngine.Close() - return unpack(newTarWalker(f), dest, ref) + return unpack(ctx, refEngine, casEngine, dest, ref) } -func unpack(w walker, dest, refName string) error { - ref, err := findDescriptor(w, refName) +func unpack(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, dest, ref string) error { + descriptor, err := refEngine.Get(ctx, ref) + if err != nil { + return errors.Wrapf(err, "failed to fetch %q", ref) + } + + err = validateMediaType(descriptor.MediaType, validRefMediaTypes) if err != nil { return err } - if err = ref.validate(w, validRefMediaTypes); err != nil { + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { return err } - m, err := findManifest(w, ref) + m, err := findManifest(ctx, casEngine, descriptor) if err != nil { return err } - if err = m.validate(w); err != nil { + if err = validateManifest(ctx, m, casEngine); err != nil { return err } - return m.unpack(w, dest) + return unpackManifest(ctx, m, casEngine, dest) } -// CreateRuntimeBundleLayout walks through the file tree given by src and -// creates an OCI runtime bundle in the given destination dest -// or returns an error if the unpacking failed. -func CreateRuntimeBundleLayout(src, dest, ref, root string) error { - return createRuntimeBundle(newPathWalker(src), dest, ref, root) -} +// CreateRuntimeBundle creates an OCI runtime bundle in the given +// destination. +func CreateRuntimeBundle(ctx context.Context, path, dest, ref, rootfs string) error { + refEngine, err := refslayout.NewEngine(ctx, path) + if err != nil { + return err + } + defer refEngine.Close() -// CreateRuntimeBundle walks through the given .tar file and -// creates an OCI runtime bundle in the given destination dest -// or returns an error if the unpacking failed. -func CreateRuntimeBundle(tarFile, dest, ref, root string) error { - f, err := os.Open(tarFile) + casEngine, err := caslayout.NewEngine(ctx, path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer casEngine.Close() - return createRuntimeBundle(newTarWalker(f), dest, ref, root) + return createRuntimeBundle(ctx, refEngine, casEngine, dest, ref, rootfs) } -func createRuntimeBundle(w walker, dest, refName, rootfs string) error { - ref, err := findDescriptor(w, refName) +func createRuntimeBundle(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, dest, ref, rootfs string) error { + descriptor, err := refEngine.Get(ctx, ref) + if err != nil { + return errors.Wrapf(err, "failed to fetch %q", ref) + } + + err = validateMediaType(descriptor.MediaType, validRefMediaTypes) if err != nil { return err } - if err = ref.validate(w, validRefMediaTypes); err != nil { + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { return err } - m, err := findManifest(w, ref) + m, err := findManifest(ctx, casEngine, descriptor) if err != nil { return err } - if err = m.validate(w); err != nil { + if err = validateManifest(ctx, m, casEngine); err != nil { return err } - c, err := findConfig(w, &m.Config) + c, err := findConfig(ctx, casEngine, &m.Config) if err != nil { return err } - err = m.unpack(w, filepath.Join(dest, rootfs)) + err = unpackManifest(ctx, m, casEngine, filepath.Join(dest, rootfs)) if err != nil { return err } - spec, err := c.runtimeSpec(rootfs) + spec, err := runtimeSpec(c, rootfs) if err != nil { return err } diff --git a/image/manifest.go b/image/manifest.go index 641e849..80323f9 100644 --- a/image/manifest.go +++ b/image/manifest.go @@ -28,93 +28,86 @@ import ( "time" "github.com/opencontainers/image-spec/schema" + "github.com/opencontainers/image-spec/specs-go" "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/image-tools/image/cas" "github.com/pkg/errors" + "golang.org/x/net/context" ) -type manifest struct { - Config descriptor `json:"config"` - Layers []descriptor `json:"layers"` -} - -func findManifest(w walker, d *descriptor) (*manifest, error) { - var m manifest - mpath := filepath.Join("blobs", d.algo(), d.hash()) - - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != mpath { - return nil - } - - buf, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrapf(err, "%s: error reading manifest", path) - } - - if err := schema.MediaTypeManifest.Validate(bytes.NewReader(buf)); err != nil { - return errors.Wrapf(err, "%s: manifest validation failed", path) - } +func findManifest(ctx context.Context, engine cas.Engine, descriptor *specs.Descriptor) (*v1.Manifest, error) { + reader, err := engine.Get(ctx, descriptor.Digest) + if err != nil { + return nil, errors.Wrapf(err, "failed to fetch %s", descriptor.Digest) + } - if err := json.Unmarshal(buf, &m); err != nil { - return err - } + buf, err := ioutil.ReadAll(reader) + if err != nil { + return nil, errors.Wrapf(err, "%s: error reading manifest", descriptor.Digest) + } - if len(m.Layers) == 0 { - return fmt.Errorf("%s: no layers found", path) - } + if err := schema.MediaTypeManifest.Validate(bytes.NewReader(buf)); err != nil { + return nil, errors.Wrapf(err, "%s: manifest validation failed", descriptor.Digest) + } - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: manifest not found", mpath) - case errEOW: - return &m, nil - default: + var m v1.Manifest + if err := json.Unmarshal(buf, &m); err != nil { return nil, err } + + if len(m.Layers) == 0 { + return nil, fmt.Errorf("%s: no layers found", descriptor.Digest) + } + + return &m, nil } -func (m *manifest) validate(w walker) error { - if err := m.Config.validate(w, []string{v1.MediaTypeImageConfig}); err != nil { - return errors.Wrap(err, "config validation failed") +func validateManifest(ctx context.Context, m *v1.Manifest, engine cas.Engine) error { + _, err := findConfig(ctx, engine, &m.Config) + if err != nil { + return errors.Wrap(err, "invalid manifest config") } for _, d := range m.Layers { - if err := d.validate(w, []string{v1.MediaTypeImageLayer}); err != nil { - return errors.Wrap(err, "layer validation failed") + err = validateMediaType( + d.MediaType, + []string{ + v1.MediaTypeImageLayer, + v1.MediaTypeImageLayerNonDistributable, + }, + ) + if err != nil { + return errors.Wrap(err, "invalid layer media type") + } + err = validateDescriptor(ctx, engine, &d) + if err != nil { + return errors.Wrap(err, "invalid layer descriptor") } } return nil } -func (m *manifest) unpack(w walker, dest string) error { +func unpackManifest(ctx context.Context, m *v1.Manifest, engine cas.Engine, dest string) (err error) { for _, d := range m.Layers { - if d.MediaType != string(schema.MediaTypeImageLayer) { - continue + err = validateMediaType( + d.MediaType, + []string{ + v1.MediaTypeImageLayer, + v1.MediaTypeImageLayerNonDistributable, + }, + ) + if err != nil { + return errors.Wrap(err, "invalid layer media type") } - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() { - return nil - } - - dd, err := filepath.Rel(filepath.Join("blobs", d.algo()), filepath.Clean(path)) - if err != nil || d.hash() != dd { - return nil - } - - if err := unpackLayer(dest, r); err != nil { - return errors.Wrap(err, "error extracting layer") - } + reader, err := engine.Get(ctx, d.Digest) + if err != nil { + return errors.Wrapf(err, "failed to fetch %s", d.Digest) + } - return errEOW - }); err { - case nil: - return fmt.Errorf("%s: layer not found", dest) - case errEOW: - default: - return err + if err := unpackLayer(dest, reader); err != nil { + return errors.Wrap(err, "error extracting layer") } } return nil diff --git a/image/manifest_test.go b/image/manifest_test.go index 350ef34..f49218b 100644 --- a/image/manifest_test.go +++ b/image/manifest_test.go @@ -18,14 +18,18 @@ import ( "archive/tar" "bytes" "compress/gzip" - "crypto/sha256" - "fmt" "io" "io/ioutil" "os" "path/filepath" "strings" "testing" + + "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-spec/specs-go/v1" + caslayout "github.com/opencontainers/image-tools/image/cas/layout" + imagelayout "github.com/opencontainers/image-tools/image/layout" + "golang.org/x/net/context" ) func TestUnpackLayerDuplicateEntries(t *testing.T) { @@ -66,53 +70,48 @@ func TestUnpackLayerDuplicateEntries(t *testing.T) { } func TestUnpackLayer(t *testing.T) { + ctx := context.Background() + tmp1, err := ioutil.TempDir("", "test-layer") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmp1) - err = os.MkdirAll(filepath.Join(tmp1, "blobs", "sha256"), 0700) + + path := filepath.Join(tmp1, "image.tar") + err = imagelayout.CreateTarFile(ctx, path) if err != nil { t.Fatal(err) } - tarfile := filepath.Join(tmp1, "blobs", "sha256", "test.tar") - f, err := os.Create(tarfile) + + engine, err := caslayout.NewEngine(ctx, path) if err != nil { t.Fatal(err) } + defer engine.Close() - gw := gzip.NewWriter(f) + var buffer bytes.Buffer + gw := gzip.NewWriter(&buffer) tw := tar.NewWriter(gw) tw.WriteHeader(&tar.Header{Name: "test", Size: 4, Mode: 0600}) io.Copy(tw, bytes.NewReader([]byte("test"))) tw.Close() gw.Close() - f.Close() - // generate sha256 hash - h := sha256.New() - file, err := os.Open(tarfile) - if err != nil { - t.Fatal(err) - } - defer file.Close() - _, err = io.Copy(h, file) - if err != nil { - t.Fatal(err) - } - err = os.Rename(tarfile, filepath.Join(tmp1, "blobs", "sha256", fmt.Sprintf("%x", h.Sum(nil)))) + digest, err := engine.Put(ctx, &buffer) if err != nil { t.Fatal(err) } - testManifest := manifest{ - Layers: []descriptor{descriptor{ + testManifest := v1.Manifest{ + Layers: []specs.Descriptor{specs.Descriptor{ MediaType: "application/vnd.oci.image.layer.tar+gzip", - Digest: fmt.Sprintf("sha256:%s", fmt.Sprintf("%x", h.Sum(nil))), + Digest: digest, }}, } - err = testManifest.unpack(newPathWalker(tmp1), filepath.Join(tmp1, "rootfs")) + + err = unpackManifest(ctx, &testManifest, engine, filepath.Join(tmp1, "rootfs")) if err != nil { t.Fatal(err) } diff --git a/image/walker.go b/image/walker.go deleted file mode 100644 index 33f9674..0000000 --- a/image/walker.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2016 The Linux Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package image - -import ( - "archive/tar" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -var ( - errEOW = fmt.Errorf("end of walk") // error to signal stop walking -) - -// walkFunc is a function type that gets called for each file or directory visited by the Walker. -type walkFunc func(path string, _ os.FileInfo, _ io.Reader) error - -// walker is the interface that walks through a file tree, -// calling walk for each file or directory in the tree. -type walker interface { - walk(walkFunc) error -} - -type tarWalker struct { - r io.ReadSeeker -} - -// newTarWalker returns a Walker that walks through .tar files. -func newTarWalker(r io.ReadSeeker) walker { - return &tarWalker{r} -} - -func (w *tarWalker) walk(f walkFunc) error { - if _, err := w.r.Seek(0, os.SEEK_SET); err != nil { - return errors.Wrapf(err, "unable to reset") - } - - tr := tar.NewReader(w.r) - -loop: - for { - hdr, err := tr.Next() - switch err { - case io.EOF: - break loop - case nil: - // success, continue below - default: - return errors.Wrapf(err, "error advancing tar stream") - } - - info := hdr.FileInfo() - if err := f(hdr.Name, info, tr); err != nil { - return err - } - } - - return nil -} - -type eofReader struct{} - -func (eofReader) Read(_ []byte) (int, error) { - return 0, io.EOF -} - -type pathWalker struct { - root string -} - -// newPathWalker returns a Walker that walks through directories -// starting at the given root path. It does not follow symlinks. -func newPathWalker(root string) walker { - return &pathWalker{root} -} - -func (w *pathWalker) walk(f walkFunc) error { - return filepath.Walk(w.root, func(path string, info os.FileInfo, err error) error { - // MUST check error value, to make sure the `os.FileInfo` is available. - // Otherwise panic risk will exist. - if err != nil { - return errors.Wrap(err, "error walking path") - } - - rel, err := filepath.Rel(w.root, path) - if err != nil { - return errors.Wrap(err, "error walking path") // err from filepath.Walk includes path name - } - - if info.IsDir() { // behave like a tar reader for directories - return f(rel, info, eofReader{}) - } - - file, err := os.Open(path) - if err != nil { - return errors.Wrap(err, "unable to open file") // os.Open includes the path - } - defer file.Close() - - return f(rel, info, file) - }) -}