diff --git a/registry/remote/repository.go b/registry/remote/repository.go index 0f8c6acd..337b4115 100644 --- a/registry/remote/repository.go +++ b/registry/remote/repository.go @@ -1425,26 +1425,25 @@ func (s *manifestStore) indexReferrersForPush(ctx context.Context, desc ocispec. func (s *manifestStore) updateReferrersIndex(ctx context.Context, subject ocispec.Descriptor, change referrerChange) (err error) { referrersTag := buildReferrersTag(subject) - skipDelete := s.repo.SkipReferrersGC - var oldIndexDesc ocispec.Descriptor - var referrers []ocispec.Descriptor + var oldIndexDesc *ocispec.Descriptor + var oldReferrers []ocispec.Descriptor prepare := func() error { // 1. pull the original referrers list using the referrers tag schema - var err error - oldIndexDesc, referrers, err = s.repo.referrersFromIndex(ctx, referrersTag) + indexDesc, referrers, err := s.repo.referrersFromIndex(ctx, referrersTag) if err != nil { if errors.Is(err, errdef.ErrNotFound) { - // no old index found, skip delete - skipDelete = true + // valid case: no old referrers index return nil } return err } + oldIndexDesc = &indexDesc + oldReferrers = referrers return nil } update := func(referrerChanges []referrerChange) error { // 2. apply the referrer changes on the referrers list - updatedReferrers, err := applyReferrerChanges(referrers, referrerChanges) + updatedReferrers, err := applyReferrerChanges(oldReferrers, referrerChanges) if err != nil { if err == errNoReferrerUpdate { return nil @@ -1453,7 +1452,12 @@ func (s *manifestStore) updateReferrersIndex(ctx context.Context, subject ocispe } // 3. push the updated referrers list using referrers tag schema - if len(updatedReferrers) > 0 { + if len(updatedReferrers) > 0 || s.repo.SkipReferrersGC { + // push a new index in either case: + // 1. the referrers list has been updated with a non-zero size + // 2. OR the updated referrers list is empty but referrers GC + // is skipped, in this case an empty index should still be pushed + // as the old index won't get deleted newIndexDesc, newIndex, err := generateIndex(updatedReferrers) if err != nil { return fmt.Errorf("failed to generate referrers index for referrers tag %s: %w", referrersTag, err) @@ -1463,14 +1467,15 @@ func (s *manifestStore) updateReferrersIndex(ctx context.Context, subject ocispe } } - // 4. delete the dangling original referrers index - if !skipDelete { - if err := s.repo.delete(ctx, oldIndexDesc, true); err != nil { - return &ReferrersError{ - Op: opDeleteReferrersIndex, - Err: fmt.Errorf("failed to delete dangling referrers index %s for referrers tag %s: %w", oldIndexDesc.Digest.String(), referrersTag, err), - Subject: subject, - } + // 4. delete the dangling original referrers index, if applicable + if s.repo.SkipReferrersGC || oldIndexDesc == nil { + return nil + } + if err := s.repo.delete(ctx, *oldIndexDesc, true); err != nil { + return &ReferrersError{ + Op: opDeleteReferrersIndex, + Err: fmt.Errorf("failed to delete dangling referrers index %s for referrers tag %s: %w", oldIndexDesc.Digest.String(), referrersTag, err), + Subject: subject, } } return nil diff --git a/registry/remote/repository_test.go b/registry/remote/repository_test.go index fdd5e9ce..3e3ee61a 100644 --- a/registry/remote/repository_test.go +++ b/registry/remote/repository_test.go @@ -3589,7 +3589,7 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { artifactDesc.ArtifactType = artifact.ArtifactType artifactDesc.Annotations = artifact.Annotations - // test pushing artifact with subject + // test pushing artifact with subject, a referrer list should be created index_1 := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, // historical value. does not pertain to OCI or docker version @@ -3671,6 +3671,92 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } + // test pushing artifact with subject when an old empty referrer list exists, + // the referrer list should be updated + emptyIndex := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + } + emptyIndexJSON, err := json.Marshal(emptyIndex) + if err != nil { + t.Error("failed to marshal index", err) + } + emptyIndexDesc := content.NewDescriptorFromBytes(emptyIndex.MediaType, emptyIndexJSON) + var indexDeleted bool + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+artifactDesc.Digest.String(): + if contentType := r.Header.Get("Content-Type"); contentType != artifactDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", artifactDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+zeroDigest: + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: + w.Write(emptyIndexJSON) + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_1.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodDelete && r.URL.Path == "/v2/test/manifests/"+emptyIndexDesc.Digest.String(): + indexDeleted = true + // no "Docker-Content-Digest" header for manifest deletion + w.WriteHeader(http.StatusAccepted) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err = url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx = context.Background() + repo, err = NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.Push(ctx, artifactDesc, bytes.NewReader(artifactJSON)) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, artifactJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(artifactJSON)) + } + if !bytes.Equal(gotReferrerIndex, indexJSON_1) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_1)) + } + if !indexDeleted { + t.Errorf("indexDeleted = %v, want %v", indexDeleted, true) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } + // test pushing image manifest with subject, referrer list should be updated manifest := ocispec.Manifest{ MediaType: ocispec.MediaTypeImageManifest, @@ -3702,7 +3788,7 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { t.Fatalf("failed to marshal manifest: %v", err) } indexDesc_2 := content.NewDescriptorFromBytes(index_2.MediaType, indexJSON_2) - var manifestDeleted bool + indexDeleted = false ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): @@ -3734,7 +3820,7 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) w.WriteHeader(http.StatusCreated) case r.Method == http.MethodDelete && r.URL.Path == "/v2/test/manifests/"+indexDesc_1.Digest.String(): - manifestDeleted = true + indexDeleted = true // no "Docker-Content-Digest" header for manifest deletion w.WriteHeader(http.StatusAccepted) default: @@ -3767,14 +3853,14 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { if !bytes.Equal(gotReferrerIndex, indexJSON_2) { t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) } - if !manifestDeleted { - t.Errorf("manifestDeleted = %v, want %v", manifestDeleted, true) + if !indexDeleted { + t.Errorf("indexDeleted = %v, want %v", indexDeleted, true) } if state := repo.loadReferrersState(); state != referrersStateUnsupported { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } - // test pushing image manifest with subject without cleaning dangling referrers + // test pushing image manifest with subject again, referrers list should not be changed ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): @@ -3792,7 +3878,91 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+zeroDigest: w.WriteHeader(http.StatusNotFound) case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: - w.Write(indexJSON_1) + w.Write(indexJSON_2) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err = url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx = context.Background() + repo, err = NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.Push(ctx, manifestDesc, bytes.NewReader(manifestJSON)) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, manifestJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) + } + // referrers list should not be changed + if !bytes.Equal(gotReferrerIndex, indexJSON_2) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } + + // push image index with subject, referrer list should be updated + indexManifest := ocispec.Index{ + MediaType: ocispec.MediaTypeImageIndex, + Subject: &subjectDesc, + ArtifactType: "test/index", + Annotations: map[string]string{"foo": "bar"}, + } + indexManifestJSON, err := json.Marshal(indexManifest) + if err != nil { + t.Fatalf("failed to marshal manifest: %v", err) + } + indexManifestDesc := content.NewDescriptorFromBytes(indexManifest.MediaType, indexManifestJSON) + indexManifestDesc.ArtifactType = indexManifest.ArtifactType + indexManifestDesc.Annotations = indexManifest.Annotations + index_3 := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{ + artifactDesc, + manifestDesc, + indexManifestDesc, + }, + } + indexJSON_3, err := json.Marshal(index_3) + if err != nil { + t.Fatalf("failed to marshal manifest: %v", err) + } + indexDesc_3 := content.NewDescriptorFromBytes(index_3.MediaType, indexJSON_3) + indexDeleted = false + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+indexManifestDesc.Digest.String(): + if contentType := r.Header.Get("Content-Type"); contentType != indexManifestDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexManifestDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+zeroDigest: + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: + w.Write(indexJSON_2) case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { w.WriteHeader(http.StatusBadRequest) @@ -3803,10 +3973,10 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { t.Errorf("fail to read: %v", err) } gotReferrerIndex = buf.Bytes() - w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) + w.Header().Set("Docker-Content-Digest", indexDesc_3.Digest.String()) w.WriteHeader(http.StatusCreated) - case r.Method == http.MethodDelete && r.URL.Path == "/v2/test/manifests/"+indexDesc_1.Digest.String(): - manifestDeleted = true + case r.Method == http.MethodDelete && r.URL.Path == "/v2/test/manifests/"+indexDesc_2.Digest.String(): + indexDeleted = true // no "Docker-Content-Digest" header for manifest deletion w.WriteHeader(http.StatusAccepted) default: @@ -3826,11 +3996,117 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { t.Fatalf("NewRepository() error = %v", err) } repo.PlainHTTP = true + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = repo.Push(ctx, indexManifestDesc, bytes.NewReader(indexManifestJSON)) + if err != nil { + t.Fatalf("Manifests.Push() error = %v", err) + } + if !bytes.Equal(gotManifest, indexManifestJSON) { + t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(indexManifestJSON)) + } + if !bytes.Equal(gotReferrerIndex, indexJSON_3) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_3)) + } + if !indexDeleted { + t.Errorf("indexDeleted = %v, want %v", indexDeleted, true) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } +} + +func Test_ManifestStore_Push_ReferrersAPIUnavailable_SkipReferrersGC(t *testing.T) { + // generate test content + subject := []byte(`{"layers":[]}`) + subjectDesc := content.NewDescriptorFromBytes(spec.MediaTypeArtifactManifest, subject) + referrersTag := strings.Replace(subjectDesc.Digest.String(), ":", "-", 1) + manifest := ocispec.Manifest{ + MediaType: ocispec.MediaTypeImageManifest, + Config: ocispec.Descriptor{ + MediaType: "testconfig", + }, + Subject: &subjectDesc, + Annotations: map[string]string{"foo": "bar"}, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("failed to marshal manifest: %v", err) + } + manifestDesc := content.NewDescriptorFromBytes(manifest.MediaType, manifestJSON) + manifestDesc.ArtifactType = manifest.Config.MediaType + manifestDesc.Annotations = manifest.Annotations + index_1 := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{ + manifestDesc, + }, + } + + // test pushing image manifest with subject, a referrers list should be created + indexJSON_1, err := json.Marshal(index_1) + if err != nil { + t.Fatalf("failed to marshal manifest: %v", err) + } + indexDesc_1 := content.NewDescriptorFromBytes(index_1.MediaType, indexJSON_1) + var gotManifest []byte + var gotReferrerIndex []byte + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): + if contentType := r.Header.Get("Content-Type"); contentType != manifestDesc.MediaType { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", manifestDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+zeroDigest: + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_1.Digest.String()) + w.WriteHeader(http.StatusCreated) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + + ctx := context.Background() + repo, err := NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true repo.SkipReferrersGC = true + if state := repo.loadReferrersState(); state != referrersStateUnknown { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) } - manifestDeleted = false err = repo.Push(ctx, manifestDesc, bytes.NewReader(manifestJSON)) if err != nil { t.Fatalf("Manifests.Push() error = %v", err) @@ -3838,17 +4114,25 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { if !bytes.Equal(gotManifest, manifestJSON) { t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) } - if !bytes.Equal(gotReferrerIndex, indexJSON_2) { - t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) - } - if manifestDeleted { - t.Errorf("manifestDeleted = %v, want %v", manifestDeleted, false) + if !bytes.Equal(gotReferrerIndex, indexJSON_1) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_1)) } if state := repo.loadReferrersState(); state != referrersStateUnsupported { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } - // test pushing image manifest with subject again, referrers list should not be changed + // test pushing image manifest with subject when an old empty referrer list exists, + // the referrer list should be updated + emptyIndex := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + } + emptyIndexJSON, err := json.Marshal(emptyIndex) + if err != nil { + t.Error("failed to marshal index", err) + } ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): @@ -3860,13 +4144,25 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { if _, err := buf.ReadFrom(r.Body); err != nil { t.Errorf("fail to read: %v", err) } - gotManifest = buf.Bytes() - w.Header().Set("Docker-Content-Digest", manifestDesc.Digest.String()) + gotManifest = buf.Bytes() + w.Header().Set("Docker-Content-Digest", manifestDesc.Digest.String()) + w.WriteHeader(http.StatusCreated) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+zeroDigest: + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: + w.Write(emptyIndexJSON) + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_1.Digest.String()) w.WriteHeader(http.StatusCreated) - case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+zeroDigest: - w.WriteHeader(http.StatusNotFound) - case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: - w.Write(indexJSON_2) default: t.Errorf("unexpected access: %s %s", r.Method, r.URL) w.WriteHeader(http.StatusNotFound) @@ -3884,6 +4180,8 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { t.Fatalf("NewRepository() error = %v", err) } repo.PlainHTTP = true + repo.SkipReferrersGC = true + if state := repo.loadReferrersState(); state != referrersStateUnknown { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) } @@ -3894,15 +4192,15 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { if !bytes.Equal(gotManifest, manifestJSON) { t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(manifestJSON)) } - // referrers list should not be changed - if !bytes.Equal(gotReferrerIndex, indexJSON_2) { - t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) + if !bytes.Equal(gotReferrerIndex, indexJSON_1) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_1)) } if state := repo.loadReferrersState(); state != referrersStateUnsupported { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } - // push image index with subject, referrer list should be updated + // push image index with subject, referrer list should be updated, the old + // one should not be deleted indexManifest := ocispec.Index{ MediaType: ocispec.MediaTypeImageIndex, Subject: &subjectDesc, @@ -3916,23 +4214,21 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { indexManifestDesc := content.NewDescriptorFromBytes(indexManifest.MediaType, indexManifestJSON) indexManifestDesc.ArtifactType = indexManifest.ArtifactType indexManifestDesc.Annotations = indexManifest.Annotations - index_3 := ocispec.Index{ + index_2 := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, // historical value. does not pertain to OCI or docker version }, MediaType: ocispec.MediaTypeImageIndex, Manifests: []ocispec.Descriptor{ - artifactDesc, manifestDesc, indexManifestDesc, }, } - indexJSON_3, err := json.Marshal(index_3) + indexJSON_2, err := json.Marshal(index_2) if err != nil { t.Fatalf("failed to marshal manifest: %v", err) } - indexDesc_3 := content.NewDescriptorFromBytes(index_3.MediaType, indexJSON_3) - manifestDeleted = false + indexDesc_2 := content.NewDescriptorFromBytes(index_2.MediaType, indexJSON_2) ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+indexManifestDesc.Digest.String(): @@ -3950,7 +4246,7 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+zeroDigest: w.WriteHeader(http.StatusNotFound) case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: - w.Write(indexJSON_2) + w.Write(indexJSON_1) case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { w.WriteHeader(http.StatusBadRequest) @@ -3961,12 +4257,8 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { t.Errorf("fail to read: %v", err) } gotReferrerIndex = buf.Bytes() - w.Header().Set("Docker-Content-Digest", indexDesc_3.Digest.String()) + w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) w.WriteHeader(http.StatusCreated) - case r.Method == http.MethodDelete && r.URL.Path == "/v2/test/manifests/"+indexDesc_2.Digest.String(): - manifestDeleted = true - // no "Docker-Content-Digest" header for manifest deletion - w.WriteHeader(http.StatusAccepted) default: t.Errorf("unexpected access: %s %s", r.Method, r.URL) w.WriteHeader(http.StatusNotFound) @@ -3984,6 +4276,8 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { t.Fatalf("NewRepository() error = %v", err) } repo.PlainHTTP = true + repo.SkipReferrersGC = true + if state := repo.loadReferrersState(); state != referrersStateUnknown { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) } @@ -3994,11 +4288,8 @@ func Test_ManifestStore_Push_ReferrersAPIUnavailable(t *testing.T) { if !bytes.Equal(gotManifest, indexManifestJSON) { t.Errorf("Manifests.Push() = %v, want %v", string(gotManifest), string(indexManifestJSON)) } - if !bytes.Equal(gotReferrerIndex, indexJSON_3) { - t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_3)) - } - if !manifestDeleted { - t.Errorf("manifestDeleted = %v, want %v", manifestDeleted, true) + if !bytes.Equal(gotReferrerIndex, indexJSON_2) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) } if state := repo.loadReferrersState(); state != referrersStateUnsupported { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) @@ -4317,7 +4608,6 @@ func Test_ManifestStore_Delete_ReferrersAPIUnavailable(t *testing.T) { } indexManifestDesc := content.NewDescriptorFromBytes(indexManifest.MediaType, indexManifestJSON) - // test deleting artifact with subject index_1 := ocispec.Index{ Versioned: specs.Versioned{ SchemaVersion: 2, // historical value. does not pertain to OCI or docker version @@ -4364,6 +4654,7 @@ func Test_ManifestStore_Delete_ReferrersAPIUnavailable(t *testing.T) { } indexDesc_3 := content.NewDescriptorFromBytes(index_3.MediaType, indexJSON_3) + // test deleting artifact with subject, referrers list should be updated manifestDeleted := false indexDeleted := false var gotReferrerIndex []byte @@ -4422,7 +4713,6 @@ func Test_ManifestStore_Delete_ReferrersAPIUnavailable(t *testing.T) { store := repo.Manifests() ctx := context.Background() - // test deleting artifact with subject if state := repo.loadReferrersState(); state != referrersStateUnknown { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) } @@ -4443,7 +4733,7 @@ func Test_ManifestStore_Delete_ReferrersAPIUnavailable(t *testing.T) { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } - // test deleting manifest with subject + // test deleting manifest with subject, referrers list should be updated manifestDeleted = false indexDeleted = false ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -4517,7 +4807,7 @@ func Test_ManifestStore_Delete_ReferrersAPIUnavailable(t *testing.T) { t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) } - // test deleting index with a subject + // test deleting index with a subject, referrers list should be updated manifestDeleted = false indexDeleted = false ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -4580,6 +4870,219 @@ func Test_ManifestStore_Delete_ReferrersAPIUnavailable(t *testing.T) { } } +func Test_ManifestStore_Delete_ReferrersAPIUnavailable_SkipReferrersGC(t *testing.T) { + // generate test content + subject := []byte(`{"layers":[]}`) + subjectDesc := content.NewDescriptorFromBytes(spec.MediaTypeArtifactManifest, subject) + referrersTag := strings.Replace(subjectDesc.Digest.String(), ":", "-", 1) + + manifest := ocispec.Manifest{ + MediaType: ocispec.MediaTypeImageManifest, + Subject: &subjectDesc, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatalf("failed to marshal manifest: %v", err) + } + manifestDesc := content.NewDescriptorFromBytes(manifest.MediaType, manifestJSON) + + indexManifest := ocispec.Index{ + MediaType: ocispec.MediaTypeImageIndex, + Subject: &subjectDesc, + } + indexManifestJSON, err := json.Marshal(indexManifest) + if err != nil { + t.Fatalf("failed to marshal manifest: %v", err) + } + indexManifestDesc := content.NewDescriptorFromBytes(indexManifest.MediaType, indexManifestJSON) + + index_1 := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{ + manifestDesc, + indexManifestDesc, + }, + } + indexJSON_1, err := json.Marshal(index_1) + if err != nil { + t.Fatalf("failed to marshal manifest: %v", err) + } + index_2 := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{ + indexManifestDesc, + }, + } + indexJSON_2, err := json.Marshal(index_2) + if err != nil { + t.Fatalf("failed to marshal manifest: %v", err) + } + indexDesc_2 := content.NewDescriptorFromBytes(index_2.MediaType, indexJSON_2) + index_3 := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + MediaType: ocispec.MediaTypeImageIndex, + Manifests: []ocispec.Descriptor{}, + } + indexJSON_3, err := json.Marshal(index_3) + if err != nil { + t.Fatalf("failed to marshal manifest: %v", err) + } + indexDesc_3 := content.NewDescriptorFromBytes(index_3.MediaType, indexJSON_3) + + // test deleting image manifest with subject, referrers list should be updated, + // the old one should not be deleted + manifestDeleted := false + var gotReferrerIndex []byte + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodDelete && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): + manifestDeleted = true + // no "Docker-Content-Digest" header for manifest deletion + w.WriteHeader(http.StatusAccepted) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+manifestDesc.Digest.String(): + if accept := r.Header.Get("Accept"); !strings.Contains(accept, manifestDesc.MediaType) { + t.Errorf("manifest not convertable: %s", accept) + w.WriteHeader(http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", manifestDesc.MediaType) + w.Header().Set("Docker-Content-Digest", manifestDesc.Digest.String()) + if _, err := w.Write(manifestJSON); err != nil { + t.Errorf("failed to write %q: %v", r.URL, err) + } + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+zeroDigest: + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: + w.Write(indexJSON_1) + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_2.Digest.String()) + w.WriteHeader(http.StatusCreated) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + repo, err := NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + repo.SkipReferrersGC = true + store := repo.Manifests() + ctx := context.Background() + + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = store.Delete(ctx, manifestDesc) + if err != nil { + t.Fatalf("Manifests.Delete() error = %v", err) + } + if !manifestDeleted { + t.Errorf("Manifests.Delete() = %v, want %v", manifestDeleted, true) + } + if !bytes.Equal(gotReferrerIndex, indexJSON_2) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_2)) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } + + // test deleting index with a subject, referrers list should be updated, + // the old one should not be deleted, an empty one should be pushed + manifestDeleted = false + ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodDelete && r.URL.Path == "/v2/test/manifests/"+indexManifestDesc.Digest.String(): + manifestDeleted = true + // no "Docker-Content-Digest" header for manifest deletion + w.WriteHeader(http.StatusAccepted) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+indexManifestDesc.Digest.String(): + if accept := r.Header.Get("Accept"); !strings.Contains(accept, indexManifestDesc.MediaType) { + t.Errorf("manifest not convertable: %s", accept) + w.WriteHeader(http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", indexManifestDesc.MediaType) + w.Header().Set("Docker-Content-Digest", indexManifestDesc.Digest.String()) + if _, err := w.Write(indexManifestJSON); err != nil { + t.Errorf("failed to write %q: %v", r.URL, err) + } + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/referrers/"+zeroDigest: + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && r.URL.Path == "/v2/test/manifests/"+referrersTag: + w.Write(indexJSON_2) + case r.Method == http.MethodPut && r.URL.Path == "/v2/test/manifests/"+referrersTag: + if contentType := r.Header.Get("Content-Type"); contentType != ocispec.MediaTypeImageIndex { + w.WriteHeader(http.StatusBadRequest) + break + } + buf := bytes.NewBuffer(nil) + if _, err := buf.ReadFrom(r.Body); err != nil { + t.Errorf("fail to read: %v", err) + } + gotReferrerIndex = buf.Bytes() + w.Header().Set("Docker-Content-Digest", indexDesc_3.Digest.String()) + w.WriteHeader(http.StatusCreated) + default: + t.Errorf("unexpected access: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + defer ts.Close() + uri, err = url.Parse(ts.URL) + if err != nil { + t.Fatalf("invalid test http server: %v", err) + } + repo, err = NewRepository(uri.Host + "/test") + if err != nil { + t.Fatalf("NewRepository() error = %v", err) + } + repo.PlainHTTP = true + repo.SkipReferrersGC = true + store = repo.Manifests() + ctx = context.Background() + + if state := repo.loadReferrersState(); state != referrersStateUnknown { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnknown) + } + err = store.Delete(ctx, indexManifestDesc) + if err != nil { + t.Fatalf("Manifests.Delete() error = %v", err) + } + if !manifestDeleted { + t.Errorf("Manifests.Delete() = %v, want %v", manifestDeleted, true) + } + if !bytes.Equal(gotReferrerIndex, indexJSON_3) { + t.Errorf("got referrers index = %v, want %v", string(gotReferrerIndex), string(indexJSON_3)) + } + if state := repo.loadReferrersState(); state != referrersStateUnsupported { + t.Errorf("Repository.loadReferrersState() = %v, want %v", state, referrersStateUnsupported) + } +} + func Test_ManifestStore_Delete_ReferrersAPIUnavailable_InconsistentIndex(t *testing.T) { // generate test content subject := []byte(`{"layers":[]}`)