diff --git a/cmd/opm/index/add.go b/cmd/opm/index/add.go index 0044017c6..3c958fe93 100644 --- a/cmd/opm/index/add.go +++ b/cmd/opm/index/add.go @@ -18,7 +18,13 @@ var ( This command will add the given set of bundle images (specified by the --bundles option) to an index image (provided by the --from-index option). - If multiple bundles are given with '--mode=replaces' (the default), bundles are added to the index by order of ascending (semver) version unless the update graph specified by replaces requires a different input order; e.g. 1.0.0 replaces 1.0.1 would result in [1.0.1, 1.0.0] instead of the [1.0.0, 1.0.1] normally expected of semver. However, for most cases (e.g. 1.0.1 replaces 1.0.0) the bundle with the highest version is used to set the default channel of the related package. + If multiple bundles are given with '--mode=replaces' (the default), bundles are added to the index by order of ascending (semver) version unless the update graph specified by replaces requires a different input order; e.g. 1.0.0 replaces 1.0.1 would result in [1.0.1, 1.0.0] instead of the [1.0.0, 1.0.1] normally expected of semver. However, for most cases (e.g. 1.0.1 replaces 1.0.0) the bundle with the highest version is used to set the default channel of the related package. + + Caveat: in replaces mode, the head of a channel is always the bundle with the highest semver. Any bundles upgrading from this channel-head will be pruned. + An upgrade graph that looks like: + 0.1.1 -> 0.1.2 -> 0.1.2-1 + will be pruned on add to: + 0.1.1 -> 0.1.2 `) addExample = templates.Examples(` diff --git a/pkg/lib/registry/registry.go b/pkg/lib/registry/registry.go index 4232e07b0..56cc98755 100644 --- a/pkg/lib/registry/registry.go +++ b/pkg/lib/registry/registry.go @@ -196,7 +196,16 @@ func populate(ctx context.Context, loader registry.Load, graphLoader registry.Gr } populator := registry.NewDirectoryPopulator(loader, graphLoader, querier, unpackedImageMap, overwriteImageMap, overwrite) - return populator.Populate(mode) + if err := populator.Populate(mode); err != nil { + return err + } + + for _, imgMap := range overwriteImageMap { + for to, from := range imgMap { + unpackedImageMap[to] = from + } + } + return checkForBundles(ctx, querier.(*sqlite.SQLQuerier), graphLoader, unpackedImageMap) } type DeleteFromRegistryRequest struct { @@ -402,3 +411,97 @@ func checkForBundlePaths(querier registry.GRPCQuery, bundlePaths []string) ([]st } return found, missing, nil } + +// packagesFromUnpackedRefs creates packages from a set of unpacked ref dirs without their upgrade edges. +func packagesFromUnpackedRefs(bundles map[image.Reference]string) (map[string]registry.Package, error) { + graph := map[string]registry.Package{} + for to, from := range bundles { + b, err := registry.NewImageInput(to, from) + if err != nil { + return nil, fmt.Errorf("failed to parse unpacked bundle image %s: %v", to, err) + } + v, err := b.Bundle.Version() + if err != nil { + return nil, fmt.Errorf("failed to parse version for %s (%s): %v", b.Bundle.Name, b.Bundle.BundleImage, err) + } + key := registry.BundleKey{ + CsvName: b.Bundle.Name, + Version: v, + BundlePath: b.Bundle.BundleImage, + } + if _, ok := graph[b.Bundle.Package]; !ok { + graph[b.Bundle.Package] = registry.Package{ + Name: b.Bundle.Package, + Channels: map[string]registry.Channel{}, + } + } + for _, c := range b.Bundle.Channels { + if _, ok := graph[b.Bundle.Package].Channels[c]; !ok { + graph[b.Bundle.Package].Channels[c] = registry.Channel{ + Nodes: map[registry.BundleKey]map[registry.BundleKey]struct{}{}, + } + } + graph[b.Bundle.Package].Channels[c].Nodes[key] = nil + } + } + + return graph, nil +} + +// replaces mode selects highest version as channel head and +// prunes any bundles in the upgrade chain after the channel head. +// check for the presence of all bundles after a replaces-mode add. +func checkForBundles(ctx context.Context, q *sqlite.SQLQuerier, g registry.GraphLoader, bundles map[image.Reference]string) error { + if len(bundles) == 0 { + return nil + } + + required, err := packagesFromUnpackedRefs(bundles) + if err != nil { + return err + } + + var errs []error + for _, pkg := range required { + graph, err := g.Generate(pkg.Name) + if err != nil { + errs = append(errs, fmt.Errorf("unable to verify added bundles for package %s: %v", pkg.Name, err)) + continue + } + + for channel, missing := range pkg.Channels { + // trace replaces chain for reachable bundles + for next := []registry.BundleKey{graph.Channels[channel].Head}; len(next) > 0; next = next[1:] { + delete(missing.Nodes, next[0]) + for edge := range graph.Channels[channel].Nodes[next[0]] { + next = append(next, edge) + } + } + + for bundle := range missing.Nodes { + // check if bundle is deprecated. Bundles readded after deprecation should not be present in index and can be ignored. + deprecated, err := isDeprecated(ctx, q, bundle) + if err != nil { + errs = append(errs, fmt.Errorf("could not validate pruned bundle %s (%s) as deprecated: %v", bundle.CsvName, bundle.BundlePath, err)) + } + if !deprecated { + errs = append(errs, fmt.Errorf("added bundle %s pruned from package %s, channel %s: this may be due to incorrect channel head (%s)", bundle.BundlePath, pkg.Name, channel, graph.Channels[channel].Head.CsvName)) + } + } + } + } + return utilerrors.NewAggregate(errs) +} + +func isDeprecated(ctx context.Context, q *sqlite.SQLQuerier, bundle registry.BundleKey) (bool, error) { + props, err := q.GetPropertiesForBundle(ctx, bundle.CsvName, bundle.Version, bundle.BundlePath) + if err != nil { + return false, err + } + for _, prop := range props { + if prop.Type == registry.DeprecatedType { + return true, nil + } + } + return false, nil +} diff --git a/pkg/lib/registry/registry_test.go b/pkg/lib/registry/registry_test.go index 0b00de282..8af530601 100644 --- a/pkg/lib/registry/registry_test.go +++ b/pkg/lib/registry/registry_test.go @@ -2,18 +2,30 @@ package registry import ( "context" + "database/sql" "encoding/json" "errors" "fmt" + "io/ioutil" + "math/rand" + "os" + "path/filepath" "testing" "testing/fstest" + "time" "github.com/stretchr/testify/require" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" "github.com/operator-framework/operator-registry/internal/model" "github.com/operator-framework/operator-registry/internal/property" "github.com/operator-framework/operator-registry/pkg/image" + "github.com/operator-framework/operator-registry/pkg/lib/bundle" "github.com/operator-framework/operator-registry/pkg/registry" + "github.com/operator-framework/operator-registry/pkg/sqlite" + "github.com/operator-framework/operator-registry/pkg/sqlite/sqlitefakes" ) func fakeBundlePathFromName(name string) string { @@ -241,3 +253,419 @@ func TestUnpackImage(t *testing.T) { }) } } + +func init() { + rand.Seed(time.Now().UTC().UnixNano()) +} + +func CreateTestDb(t *testing.T) (*sql.DB, func()) { + dbName := fmt.Sprintf("test-%d.db", rand.Int()) + + db, err := sqlite.Open(dbName) + require.NoError(t, err) + + return db, func() { + defer func() { + if err := os.Remove(dbName); err != nil { + t.Fatal(err) + } + }() + if err := db.Close(); err != nil { + t.Fatal(err) + } + } +} + +func newUnpackedTestBundle(dir, name string, csvSpec json.RawMessage, annotations registry.Annotations) (string, func(), error) { + bundleDir := filepath.Join(dir, fmt.Sprintf("%s-%s", annotations.PackageName, name)) + cleanup := func() { + os.RemoveAll(bundleDir) + } + if err := os.Mkdir(bundleDir, 0755); err != nil { + return bundleDir, cleanup, err + } + if err := os.Mkdir(filepath.Join(bundleDir, bundle.ManifestsDir), 0755); err != nil { + return bundleDir, cleanup, err + } + if err := os.Mkdir(filepath.Join(bundleDir, bundle.MetadataDir), 0755); err != nil { + return bundleDir, cleanup, err + } + if len(csvSpec) == 0 { + csvSpec = json.RawMessage(`{}`) + } + + rawCSV, err := json.Marshal(registry.ClusterServiceVersion{ + TypeMeta: v1.TypeMeta{ + Kind: sqlite.ClusterServiceVersionKind, + }, + ObjectMeta: v1.ObjectMeta{ + Name: name, + }, + Spec: csvSpec, + }) + if err != nil { + return bundleDir, cleanup, err + } + + rawObj := unstructured.Unstructured{} + if err := json.Unmarshal(rawCSV, &rawObj); err != nil { + return bundleDir, cleanup, err + } + rawObj.SetCreationTimestamp(v1.Time{}) + + jsonout, err := rawObj.MarshalJSON() + out, err := yaml.JSONToYAML(jsonout) + if err != nil { + return bundleDir, cleanup, err + } + if err := ioutil.WriteFile(filepath.Join(bundleDir, bundle.ManifestsDir, "csv.yaml"), out, 0666); err != nil { + return bundleDir, cleanup, err + } + + out, err = yaml.Marshal(registry.AnnotationsFile{Annotations: annotations}) + if err != nil { + return bundleDir, cleanup, err + } + if err := ioutil.WriteFile(filepath.Join(bundleDir, bundle.MetadataDir, "annotations.yaml"), out, 0666); err != nil { + return bundleDir, cleanup, err + } + return bundleDir, cleanup, nil +} + +type bundleDir struct { + csvSpec json.RawMessage + annotations registry.Annotations +} + +func TestPackagesFromUnpackedRefs(t *testing.T) { + tests := []struct { + description string + bundles map[string]bundleDir + expected map[string]registry.Package + wantErr bool + }{ + { + description: "InvalidBundle/Empty", + bundles: map[string]bundleDir{ + "bundle-empty": {}, + }, + wantErr: true, + }, + { + description: "LoadPartialGraph", + bundles: map[string]bundleDir{ + "testoperator-1": { + csvSpec: json.RawMessage(`{"version":"1.1.0","replaces":"1.0.0"}`), + annotations: registry.Annotations{ + PackageName: "testpkg-1", + Channels: "alpha", + DefaultChannelName: "stable", + }, + }, + "testoperator-2": { + csvSpec: json.RawMessage(`{"version":"2.1.0"}`), + annotations: registry.Annotations{ + PackageName: "testpkg-2", + Channels: "stable,alpha", + DefaultChannelName: "stable", + }, + }, + }, + expected: map[string]registry.Package{ + "testpkg-1": { + Name: "testpkg-1", + Channels: map[string]registry.Channel{ + "alpha": { + Nodes: map[registry.BundleKey]map[registry.BundleKey]struct{}{ + registry.BundleKey{ + BundlePath: fakeBundlePathFromName("testoperator-1"), + Version: "1.1.0", + CsvName: "testoperator-1", + }: nil, + }, + }, + }, + }, + "testpkg-2": { + Name: "testpkg-2", + Channels: map[string]registry.Channel{ + "alpha": { + Nodes: map[registry.BundleKey]map[registry.BundleKey]struct{}{ + registry.BundleKey{ + BundlePath: fakeBundlePathFromName("testoperator-2"), + Version: "2.1.0", + CsvName: "testoperator-2", + }: nil, + }, + }, + "stable": { + Nodes: map[registry.BundleKey]map[registry.BundleKey]struct{}{ + registry.BundleKey{ + BundlePath: fakeBundlePathFromName("testoperator-2"), + Version: "2.1.0", + CsvName: "testoperator-2", + }: nil, + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + tmpdir, err := os.MkdirTemp(".", "tmpdir-*") + defer os.RemoveAll(tmpdir) + require.NoError(t, err) + refs := map[image.Reference]string{} + for name, b := range tt.bundles { + dir, _, err := newUnpackedTestBundle(tmpdir, name, b.csvSpec, b.annotations) + require.NoError(t, err) + refs[image.SimpleReference(fakeBundlePathFromName(name))] = dir + } + pkg, err := packagesFromUnpackedRefs(refs) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.EqualValues(t, tt.expected, pkg) + }) + } +} + +func TestCheckForBundles(t *testing.T) { + type step struct { + bundles map[string]bundleDir + action int + } + const ( + actionAdd = iota + actionDeprecate + actionOverwrite + ) + tests := []struct { + description string + steps []step + wantErr error + init func() (*sql.DB, func()) + }{ + { + // 1.1.0 -> 1.0.0 pruned channel 1 + // \-> 1.2.0 ok channel 2 + description: "partialPruning", + steps: []step{ + { + bundles: map[string]bundleDir{ + "unorderedReplaces-1.1.0": { + csvSpec: json.RawMessage(`{"version":"1.1.0"}`), + annotations: registry.Annotations{ + PackageName: "testpkg", + Channels: "stable,alpha", + DefaultChannelName: "stable", + }, + }, + "unorderedReplaces-1.0.0": { + csvSpec: json.RawMessage(`{"version":"1.0.0","replaces":"unorderedReplaces-1.1.0"}`), + annotations: registry.Annotations{ + PackageName: "testpkg", + Channels: "stable,alpha", + DefaultChannelName: "stable", + }, + }, + "unorderedReplaces-1.2.0": { + csvSpec: json.RawMessage(`{"version":"1.2.0","replaces":"unorderedReplaces-1.0.0"}`), + annotations: registry.Annotations{ + PackageName: "testpkg", + Channels: "alpha", + DefaultChannelName: "stable", + }, + }, + }, + action: actionAdd, + }, + }, + wantErr: fmt.Errorf("added bundle unorderedReplaces-1.0.0 pruned from package testpkg, channel stable: this may be due to incorrect channel head (unorderedReplaces-1.1.0)"), + }, + { + description: "ignoreDeprecated", + steps: []step{ + { + bundles: map[string]bundleDir{ + "ignoreDeprecated-1.0.0": { + csvSpec: json.RawMessage(`{"version":"1.0.0"}`), + annotations: registry.Annotations{ + PackageName: "testpkg", + Channels: "stable", + }, + }, + "ignoreDeprecated-1.1.0": { + csvSpec: json.RawMessage(`{"version":"1.1.0","replaces":"ignoreDeprecated-1.0.0"}`), + annotations: registry.Annotations{ + PackageName: "testpkg", + Channels: "stable", + }, + }, + "ignoreDeprecated-1.2.0": { + csvSpec: json.RawMessage(`{"version":"1.2.0","replaces":"ignoreDeprecated-1.1.0"}`), + annotations: registry.Annotations{ + PackageName: "testpkg", + Channels: "stable", + }, + }, + }, + action: actionAdd, + }, + { + bundles: map[string]bundleDir{ + "ignoreDeprecated-1.1.0": {}, + }, + action: actionDeprecate, + }, + { + bundles: map[string]bundleDir{ + "ignoreDeprecated-1.0.0": { + csvSpec: json.RawMessage(`{"version":"1.0.0"}`), + annotations: registry.Annotations{ + PackageName: "testpkg", + Channels: "stable", + }, + }, + "ignoreDeprecated-1.1.0": { + csvSpec: json.RawMessage(`{"version":"1.1.0","replaces":"ignoreDeprecated-1.0.0"}`), + annotations: registry.Annotations{ + PackageName: "testpkg", + Channels: "stable", + }, + }, + }, + action: actionOverwrite, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + tmpdir, err := os.MkdirTemp(".", "tmpdir-*") + defer os.RemoveAll(tmpdir) + db, cleanup := CreateTestDb(t) + defer cleanup() + load, err := sqlite.NewSQLLiteLoader(db) + require.NoError(t, err) + require.NoError(t, load.Migrate(context.TODO())) + query := sqlite.NewSQLLiteQuerierFromDb(db) + graphLoader, err := sqlite.NewSQLGraphLoaderFromDB(db) + require.NoError(t, err) + + for _, step := range tt.steps { + switch step.action { + case actionDeprecate: + for deprecate := range step.bundles { + require.NoError(t, load.DeprecateBundle(deprecate)) + } + case actionAdd: + refs := map[image.Reference]string{} + for name, b := range step.bundles { + dir, _, err := newUnpackedTestBundle(tmpdir, name, b.csvSpec, b.annotations) + require.NoError(t, err) + refs[image.SimpleReference(name)] = dir + } + require.NoError(t, registry.NewDirectoryPopulator( + load, + graphLoader, + query, + refs, + nil, + false).Populate(registry.ReplacesMode)) + + err = checkForBundles(context.TODO(), query, graphLoader, refs) + if tt.wantErr == nil { + require.NoError(t, err) + return + } + require.EqualError(t, err, tt.wantErr.Error()) + + case actionOverwrite: + overwriteRefs := map[string]map[image.Reference]string{} + refs := map[image.Reference]string{} + for name, b := range step.bundles { + dir, _, err := newUnpackedTestBundle(tmpdir, name, b.csvSpec, b.annotations) + require.NoError(t, err) + to := image.SimpleReference(name) + refs[image.SimpleReference(name)] = dir + refs[to] = dir + img, err := registry.NewImageInput(to, dir) + require.NoError(t, err) + if _, ok := overwriteRefs[img.Bundle.Package]; ok { + overwriteRefs[img.Bundle.Package] = map[image.Reference]string{} + } + overwriteRefs[img.Bundle.Package][to] = dir + } + require.NoError(t, registry.NewDirectoryPopulator( + load, + graphLoader, + query, + nil, + overwriteRefs, + true).Populate(registry.ReplacesMode)) + + err = checkForBundles(context.TODO(), query, graphLoader, refs) + if tt.wantErr == nil { + require.NoError(t, err) + return + } + require.EqualError(t, err, tt.wantErr.Error()) + } + } + }) + } +} + +func TestDeprecated(t *testing.T) { + deprecated := map[string]bool{ + "deprecatedBundle": true, + "otherBundle": false, + } + q := &sqlitefakes.FakeQuerier{ + QueryContextStub: func(ctx context.Context, query string, args ...interface{}) (sqlite.RowScanner, error) { + bundleName := args[2].(string) + if len(bundleName) == 0 { + return nil, fmt.Errorf("empty bundle name") + } + hasNext := true + return &sqlitefakes.FakeRowScanner{ScanStub: func(args ...interface{}) error { + if deprecated[bundleName] { + *args[0].(*sql.NullString) = sql.NullString{ + String: registry.DeprecatedType, + Valid: true, + } + *args[1].(*sql.NullString) = sql.NullString{ + Valid: true, + } + } + return nil + }, + NextStub: func() bool { + if hasNext { + hasNext = false + return true + } + return false + }, + }, nil + }, + } + + querier := sqlite.NewSQLLiteQuerierFromDBQuerier(q) + + _, err := isDeprecated(context.TODO(), querier, registry.BundleKey{}) + require.Error(t, err) + + for b := range deprecated { + isDeprecated, err := isDeprecated(context.TODO(), querier, registry.BundleKey{BundlePath: b}) + require.NoError(t, err) + require.Equal(t, deprecated[b], isDeprecated) + } +}