diff --git a/.github/workflows/gateway-sharness.yml b/.github/workflows/gateway-sharness.yml index f362ab994..72c36a26c 100644 --- a/.github/workflows/gateway-sharness.yml +++ b/.github/workflows/gateway-sharness.yml @@ -16,7 +16,7 @@ jobs: shell: bash steps: - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: 1.19.1 - name: Checkout boxo @@ -30,10 +30,6 @@ jobs: path: kubo - name: Install Missing Tools run: sudo apt install -y socat net-tools fish libxml2-utils - - name: Restore Go Cache - uses: protocol/cache-go-action@v1 - with: - name: ${{ github.job }} - name: Replace boxo in Kubo go.mod run: | go mod edit -replace=github.com/ipfs/boxo=../boxo diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc20232a..47359c98d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,17 @@ The following emojis are used to highlight certain changes: ### Added +* ✨ The `routing/http` implements Delegated Peer Routing introduced in [IPIP-417](https://github.com/ipfs/specs/pull/417). + ### Changed +* 🛠 The `routing/http` package received the following modifications: + * Client `GetIPNSRecord` and `PutIPNSRecord` have been renamed to `GetIPNS` and + `PutIPNS`, respectively. Similarly, the required function names in the server + `ContentRouter` have also been updated. + * `ReadBitswapProviderRecord` has been renamed to `BitswapRecord` and marked as deprecated. + From now on, please use the protocol-agnostic `PeerRecord` for most use cases. The new + Peer Schema has been introduced in [IPIP-417](https://github.com/ipfs/specs/pull/417). * `boxo/gateway` * 🛠 The `IPFSBackend` interface was updated to make the responses of the `Head` method more explicit. It now returns a `HeadResponse` instead of a @@ -25,6 +34,12 @@ The following emojis are used to highlight certain changes: ### Removed +* 🛠 The `routing/http` package experienced following removals: + * Server and client no longer support the experimental `Provide` method. + `ProvideBitswap` is still usable, but marked as deprecated. A protocol-agnostic + provide mechanism is being worked on in [IPIP-378](https://github.com/ipfs/specs/pull/378). + * Server no longer exports `FindProvidersPath` and `ProvidePath`. + ### Fixed ### Security @@ -37,7 +52,7 @@ The following emojis are used to highlight certain changes: as per [IPIP-379](https://specs.ipfs.tech/ipips/ipip-0379/). * 🛠 The `verifycid` package has been updated with the new Allowlist interface as part of reducing globals efforts. -* The `blockservice` and `provider` packages has been updated to accommodate for +* The `blockservice` and `provider` packages has been updated to accommodate for changes in `verifycid`. ### Changed diff --git a/bitswap/bitswap_test.go b/bitswap/bitswap_test.go index 02bd57947..014f75a07 100644 --- a/bitswap/bitswap_test.go +++ b/bitswap/bitswap_test.go @@ -14,7 +14,6 @@ import ( "github.com/ipfs/boxo/bitswap/server" testinstance "github.com/ipfs/boxo/bitswap/testinstance" tn "github.com/ipfs/boxo/bitswap/testnet" - "github.com/ipfs/boxo/internal/test" mockrouting "github.com/ipfs/boxo/routing/mock" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" @@ -49,8 +48,6 @@ func addBlock(t *testing.T, ctx context.Context, inst testinstance.Instance, blk const kNetworkDelay = 0 * time.Millisecond func TestClose(t *testing.T) { - test.Flaky(t) - vnet := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) ig := testinstance.NewTestInstanceGenerator(vnet, nil, nil) defer ig.Close() @@ -67,8 +64,6 @@ func TestClose(t *testing.T) { } func TestProviderForKeyButNetworkCannotFind(t *testing.T) { // TODO revisit this - test.Flaky(t) - rs := mockrouting.NewServer() net := tn.VirtualNetwork(rs, delay.Fixed(kNetworkDelay)) ig := testinstance.NewTestInstanceGenerator(net, nil, nil) @@ -94,8 +89,6 @@ func TestProviderForKeyButNetworkCannotFind(t *testing.T) { // TODO revisit this } func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) { - test.Flaky(t) - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) block := blocks.NewBlock([]byte("block")) ig := testinstance.NewTestInstanceGenerator(net, nil, nil) @@ -124,8 +117,6 @@ func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) { } func TestDoesNotProvideWhenConfiguredNotTo(t *testing.T) { - test.Flaky(t) - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) block := blocks.NewBlock([]byte("block")) bsOpts := []bitswap.Option{bitswap.ProvideEnabled(false), bitswap.ProviderSearchDelay(50 * time.Millisecond)} @@ -158,8 +149,6 @@ func TestDoesNotProvideWhenConfiguredNotTo(t *testing.T) { // Tests that a received block is not stored in the blockstore if the block was // not requested by the client func TestUnwantedBlockNotAdded(t *testing.T) { - test.Flaky(t) - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) block := blocks.NewBlock([]byte("block")) bsMessage := bsmsg.New(true) @@ -195,8 +184,6 @@ func TestUnwantedBlockNotAdded(t *testing.T) { // // (because the live request queue is full) func TestPendingBlockAdded(t *testing.T) { - test.Flaky(t) - ctx := context.Background() net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) bg := blocksutil.NewBlockGenerator() @@ -245,8 +232,6 @@ func TestPendingBlockAdded(t *testing.T) { } func TestLargeSwarm(t *testing.T) { - test.Flaky(t) - if testing.Short() { t.SkipNow() } @@ -279,8 +264,6 @@ func TestLargeFile(t *testing.T) { } func TestLargeFileTwoPeers(t *testing.T) { - test.Flaky(t) - if testing.Short() { t.SkipNow() } @@ -290,8 +273,6 @@ func TestLargeFileTwoPeers(t *testing.T) { } func PerformDistributionTest(t *testing.T, numInstances, numBlocks int) { - test.Flaky(t) - ctx := context.Background() if testing.Short() { t.SkipNow() @@ -349,8 +330,6 @@ func PerformDistributionTest(t *testing.T, numInstances, numBlocks int) { // TODO simplify this test. get to the _essence_! func TestSendToWantingPeer(t *testing.T) { - test.Flaky(t) - if testing.Short() { t.SkipNow() } @@ -393,8 +372,6 @@ func TestSendToWantingPeer(t *testing.T) { } func TestEmptyKey(t *testing.T) { - test.Flaky(t) - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) ig := testinstance.NewTestInstanceGenerator(net, nil, nil) defer ig.Close() @@ -428,8 +405,6 @@ func assertStat(t *testing.T, st *bitswap.Stat, sblks, rblks, sdata, rdata uint6 } func TestBasicBitswap(t *testing.T) { - test.Flaky(t) - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) ig := testinstance.NewTestInstanceGenerator(net, nil, nil) defer ig.Close() @@ -502,8 +477,6 @@ func TestBasicBitswap(t *testing.T) { } func TestDoubleGet(t *testing.T) { - test.Flaky(t) - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) ig := testinstance.NewTestInstanceGenerator(net, nil, nil) defer ig.Close() @@ -569,8 +542,6 @@ func TestDoubleGet(t *testing.T) { } func TestWantlistCleanup(t *testing.T) { - test.Flaky(t) - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) ig := testinstance.NewTestInstanceGenerator(net, nil, nil) defer ig.Close() @@ -693,8 +664,6 @@ func newReceipt(sent, recv, exchanged uint64) *server.Receipt { } func TestBitswapLedgerOneWay(t *testing.T) { - test.Flaky(t) - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) ig := testinstance.NewTestInstanceGenerator(net, nil, nil) defer ig.Close() @@ -744,8 +713,6 @@ func TestBitswapLedgerOneWay(t *testing.T) { } func TestBitswapLedgerTwoWay(t *testing.T) { - test.Flaky(t) - net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) ig := testinstance.NewTestInstanceGenerator(net, nil, nil) defer ig.Close() @@ -834,8 +801,6 @@ func (tsl *testingScoreLedger) Stop() { // Tests start and stop of a custom decision logic func TestWithScoreLedger(t *testing.T) { - test.Flaky(t) - tsl := newTestingScoreLedger() net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(kNetworkDelay)) bsOpts := []bitswap.Option{bitswap.WithScoreLedger(tsl)} diff --git a/bitswap/client/bitswap_with_sessions_test.go b/bitswap/client/bitswap_with_sessions_test.go index 2191a5a90..54c320236 100644 --- a/bitswap/client/bitswap_with_sessions_test.go +++ b/bitswap/client/bitswap_with_sessions_test.go @@ -11,7 +11,6 @@ import ( "github.com/ipfs/boxo/bitswap/client/traceability" testinstance "github.com/ipfs/boxo/bitswap/testinstance" tn "github.com/ipfs/boxo/bitswap/testnet" - "github.com/ipfs/boxo/internal/test" mockrouting "github.com/ipfs/boxo/routing/mock" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" @@ -40,8 +39,6 @@ func addBlock(t *testing.T, ctx context.Context, inst testinstance.Instance, blk } func TestBasicSessions(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -109,8 +106,6 @@ func assertBlockListsFrom(from peer.ID, got, exp []blocks.Block) error { } func TestSessionBetweenPeers(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -171,8 +166,6 @@ func TestSessionBetweenPeers(t *testing.T) { } func TestSessionSplitFetch(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -217,8 +210,6 @@ func TestSessionSplitFetch(t *testing.T) { } func TestFetchNotConnected(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() @@ -262,8 +253,6 @@ func TestFetchNotConnected(t *testing.T) { } func TestFetchAfterDisconnect(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() @@ -342,8 +331,6 @@ func TestFetchAfterDisconnect(t *testing.T) { } func TestInterestCacheOverflow(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -394,8 +381,6 @@ func TestInterestCacheOverflow(t *testing.T) { } func TestPutAfterSessionCacheEvict(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -434,8 +419,6 @@ func TestPutAfterSessionCacheEvict(t *testing.T) { } func TestMultipleSessions(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -477,8 +460,6 @@ func TestMultipleSessions(t *testing.T) { } func TestWantlistClearsOnCancel(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() diff --git a/bitswap/client/internal/blockpresencemanager/blockpresencemanager_test.go b/bitswap/client/internal/blockpresencemanager/blockpresencemanager_test.go index 5e30073a3..b0616e4e3 100644 --- a/bitswap/client/internal/blockpresencemanager/blockpresencemanager_test.go +++ b/bitswap/client/internal/blockpresencemanager/blockpresencemanager_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" cid "github.com/ipfs/go-cid" peer "github.com/libp2p/go-libp2p/core/peer" ) @@ -17,8 +16,6 @@ const ( ) func TestBlockPresenceManager(t *testing.T) { - test.Flaky(t) - bpm := New() p := testutil.GeneratePeers(1)[0] @@ -99,8 +96,6 @@ func TestBlockPresenceManager(t *testing.T) { } func TestAddRemoveMulti(t *testing.T) { - test.Flaky(t) - bpm := New() peers := testutil.GeneratePeers(2) @@ -184,8 +179,6 @@ func TestAddRemoveMulti(t *testing.T) { } func TestAllPeersDoNotHaveBlock(t *testing.T) { - test.Flaky(t) - bpm := New() peers := testutil.GeneratePeers(3) diff --git a/bitswap/client/internal/messagequeue/donthavetimeoutmgr_test.go b/bitswap/client/internal/messagequeue/donthavetimeoutmgr_test.go index a6a28aab1..ee90ddde9 100644 --- a/bitswap/client/internal/messagequeue/donthavetimeoutmgr_test.go +++ b/bitswap/client/internal/messagequeue/donthavetimeoutmgr_test.go @@ -9,7 +9,6 @@ import ( "github.com/benbjohnson/clock" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" cid "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/p2p/protocol/ping" ) @@ -74,8 +73,6 @@ func (tr *timeoutRecorder) clear() { } func TestDontHaveTimeoutMgrTimeout(t *testing.T) { - test.Flaky(t) - firstks := testutil.GenerateCids(2) secondks := append(firstks, testutil.GenerateCids(3)...) latency := time.Millisecond * 20 @@ -132,8 +129,6 @@ func TestDontHaveTimeoutMgrTimeout(t *testing.T) { } func TestDontHaveTimeoutMgrCancel(t *testing.T) { - test.Flaky(t) - ks := testutil.GenerateCids(3) latency := time.Millisecond * 10 latMultiplier := 1 @@ -170,8 +165,6 @@ func TestDontHaveTimeoutMgrCancel(t *testing.T) { } func TestDontHaveTimeoutWantCancelWant(t *testing.T) { - test.Flaky(t) - ks := testutil.GenerateCids(3) latency := time.Millisecond * 20 latMultiplier := 1 @@ -225,8 +218,6 @@ func TestDontHaveTimeoutWantCancelWant(t *testing.T) { } func TestDontHaveTimeoutRepeatedAddPending(t *testing.T) { - test.Flaky(t) - ks := testutil.GenerateCids(10) latency := time.Millisecond * 5 latMultiplier := 1 @@ -260,8 +251,6 @@ func TestDontHaveTimeoutRepeatedAddPending(t *testing.T) { } func TestDontHaveTimeoutMgrMessageLatency(t *testing.T) { - test.Flaky(t) - ks := testutil.GenerateCids(2) latency := time.Millisecond * 40 latMultiplier := 1 @@ -311,8 +300,6 @@ func TestDontHaveTimeoutMgrMessageLatency(t *testing.T) { } func TestDontHaveTimeoutMgrMessageLatencyMax(t *testing.T) { - test.Flaky(t) - ks := testutil.GenerateCids(2) clock := clock.NewMock() pinged := make(chan struct{}) @@ -346,8 +333,6 @@ func TestDontHaveTimeoutMgrMessageLatencyMax(t *testing.T) { } func TestDontHaveTimeoutMgrUsesDefaultTimeoutIfPingError(t *testing.T) { - test.Flaky(t) - ks := testutil.GenerateCids(2) latency := time.Millisecond * 1 latMultiplier := 2 @@ -389,8 +374,6 @@ func TestDontHaveTimeoutMgrUsesDefaultTimeoutIfPingError(t *testing.T) { } func TestDontHaveTimeoutMgrUsesDefaultTimeoutIfLatencyLonger(t *testing.T) { - test.Flaky(t) - ks := testutil.GenerateCids(2) latency := time.Millisecond * 200 latMultiplier := 1 @@ -431,8 +414,6 @@ func TestDontHaveTimeoutMgrUsesDefaultTimeoutIfLatencyLonger(t *testing.T) { } func TestDontHaveTimeoutNoTimeoutAfterShutdown(t *testing.T) { - test.Flaky(t) - ks := testutil.GenerateCids(2) latency := time.Millisecond * 10 latMultiplier := 1 diff --git a/bitswap/client/internal/messagequeue/messagequeue_test.go b/bitswap/client/internal/messagequeue/messagequeue_test.go index 886e60772..9abfa68b0 100644 --- a/bitswap/client/internal/messagequeue/messagequeue_test.go +++ b/bitswap/client/internal/messagequeue/messagequeue_test.go @@ -161,8 +161,6 @@ func expectEvent(t *testing.T, events <-chan messageEvent, expectedEvent message } func TestStartupAndShutdown(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) @@ -201,8 +199,6 @@ func TestStartupAndShutdown(t *testing.T) { } func TestSendingMessagesDeduped(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) @@ -316,8 +312,6 @@ func TestSendingMessagesPriority(t *testing.T) { } func TestCancelOverridesPendingWants(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) @@ -368,8 +362,6 @@ func TestCancelOverridesPendingWants(t *testing.T) { } func TestWantOverridesPendingCancels(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) @@ -416,8 +408,6 @@ func TestWantOverridesPendingCancels(t *testing.T) { } func TestWantlistRebroadcast(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) @@ -526,8 +516,6 @@ func TestWantlistRebroadcast(t *testing.T) { } func TestSendingLargeMessages(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) @@ -557,8 +545,6 @@ func TestSendingLargeMessages(t *testing.T) { } func TestSendToPeerThatDoesntSupportHave(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) @@ -613,8 +599,6 @@ func TestSendToPeerThatDoesntSupportHave(t *testing.T) { } func TestSendToPeerThatDoesntSupportHaveMonitorsTimeouts(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) @@ -646,8 +630,6 @@ func TestSendToPeerThatDoesntSupportHaveMonitorsTimeouts(t *testing.T) { } func TestResponseReceived(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) @@ -697,8 +679,6 @@ func TestResponseReceived(t *testing.T) { } func TestResponseReceivedAppliesForFirstResponseOnly(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) @@ -743,8 +723,6 @@ func TestResponseReceivedAppliesForFirstResponseOnly(t *testing.T) { } func TestResponseReceivedDiscardsOutliers(t *testing.T) { - test.Flaky(t) - ctx := context.Background() messagesSent := make(chan []bsmsg.Entry) resetChan := make(chan struct{}, 1) diff --git a/bitswap/client/internal/notifications/notifications_test.go b/bitswap/client/internal/notifications/notifications_test.go index b4b8ef55d..68b4da1ec 100644 --- a/bitswap/client/internal/notifications/notifications_test.go +++ b/bitswap/client/internal/notifications/notifications_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - "github.com/ipfs/boxo/internal/test" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" blocksutil "github.com/ipfs/go-ipfs-blocksutil" @@ -14,7 +13,6 @@ import ( ) func TestDuplicates(t *testing.T) { - test.Flaky(t) var zero peer.ID // this test doesn't check the peer id b1 := blocks.NewBlock([]byte("1")) @@ -42,7 +40,6 @@ func TestDuplicates(t *testing.T) { } func TestPublishSubscribe(t *testing.T) { - test.Flaky(t) var zero peer.ID // this test doesn't check the peer id blockSent := blocks.NewBlock([]byte("Greetings from The Interval")) @@ -61,7 +58,6 @@ func TestPublishSubscribe(t *testing.T) { } func TestSubscribeMany(t *testing.T) { - test.Flaky(t) var zero peer.ID // this test doesn't check the peer id e1 := blocks.NewBlock([]byte("1")) @@ -89,7 +85,6 @@ func TestSubscribeMany(t *testing.T) { // TestDuplicateSubscribe tests a scenario where a given block // would be requested twice at the same time. func TestDuplicateSubscribe(t *testing.T) { - test.Flaky(t) var zero peer.ID // this test doesn't check the peer id e1 := blocks.NewBlock([]byte("1")) @@ -114,8 +109,6 @@ func TestDuplicateSubscribe(t *testing.T) { } func TestShutdownBeforeUnsubscribe(t *testing.T) { - test.Flaky(t) - e1 := blocks.NewBlock([]byte("1")) n := New() @@ -135,8 +128,6 @@ func TestShutdownBeforeUnsubscribe(t *testing.T) { } func TestSubscribeIsANoopWhenCalledWithNoKeys(t *testing.T) { - test.Flaky(t) - n := New() defer n.Shutdown() ch := n.Subscribe(context.Background()) // no keys provided @@ -146,8 +137,6 @@ func TestSubscribeIsANoopWhenCalledWithNoKeys(t *testing.T) { } func TestCarryOnWhenDeadlineExpires(t *testing.T) { - test.Flaky(t) - impossibleDeadline := time.Nanosecond fastExpiringCtx, cancel := context.WithTimeout(context.Background(), impossibleDeadline) defer cancel() @@ -161,7 +150,6 @@ func TestCarryOnWhenDeadlineExpires(t *testing.T) { } func TestDoesNotDeadLockIfContextCancelledBeforePublish(t *testing.T) { - test.Flaky(t) var zero peer.ID // this test doesn't check the peer id g := blocksutil.NewBlockGenerator() diff --git a/bitswap/client/internal/peermanager/peermanager_test.go b/bitswap/client/internal/peermanager/peermanager_test.go index a2569201f..d3c712704 100644 --- a/bitswap/client/internal/peermanager/peermanager_test.go +++ b/bitswap/client/internal/peermanager/peermanager_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" cid "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" ) @@ -80,8 +79,6 @@ func makePeerQueueFactory(msgs chan msg) PeerQueueFactory { } func TestAddingAndRemovingPeers(t *testing.T) { - test.Flaky(t) - ctx := context.Background() msgs := make(chan msg, 16) peerQueueFactory := makePeerQueueFactory(msgs) @@ -125,8 +122,6 @@ func TestAddingAndRemovingPeers(t *testing.T) { } func TestBroadcastOnConnect(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() msgs := make(chan msg, 16) @@ -148,8 +143,6 @@ func TestBroadcastOnConnect(t *testing.T) { } func TestBroadcastWantHaves(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() msgs := make(chan msg, 16) @@ -191,8 +184,6 @@ func TestBroadcastWantHaves(t *testing.T) { } func TestSendWants(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() msgs := make(chan msg, 16) @@ -227,8 +218,6 @@ func TestSendWants(t *testing.T) { } func TestSendCancels(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() msgs := make(chan msg, 16) @@ -289,8 +278,6 @@ func newSess(id uint64) *sess { } func TestSessionRegistration(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() msgs := make(chan msg, 16) diff --git a/bitswap/client/internal/peermanager/peerwantmanager_test.go b/bitswap/client/internal/peermanager/peerwantmanager_test.go index b4c3d5029..505fbea1a 100644 --- a/bitswap/client/internal/peermanager/peerwantmanager_test.go +++ b/bitswap/client/internal/peermanager/peerwantmanager_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" cid "github.com/ipfs/go-cid" peer "github.com/libp2p/go-libp2p/core/peer" ) @@ -61,8 +60,6 @@ func clearSent(pqs map[peer.ID]PeerQueue) { } func TestEmpty(t *testing.T) { - test.Flaky(t) - pwm := newPeerWantManager(&gauge{}, &gauge{}) if len(pwm.getWantBlocks()) > 0 { @@ -74,8 +71,6 @@ func TestEmpty(t *testing.T) { } func TestPWMBroadcastWantHaves(t *testing.T) { - test.Flaky(t) - pwm := newPeerWantManager(&gauge{}, &gauge{}) peers := testutil.GeneratePeers(3) @@ -188,8 +183,6 @@ func TestPWMBroadcastWantHaves(t *testing.T) { } func TestPWMSendWants(t *testing.T) { - test.Flaky(t) - pwm := newPeerWantManager(&gauge{}, &gauge{}) peers := testutil.GeneratePeers(2) @@ -270,8 +263,6 @@ func TestPWMSendWants(t *testing.T) { } func TestPWMSendCancels(t *testing.T) { - test.Flaky(t) - pwm := newPeerWantManager(&gauge{}, &gauge{}) peers := testutil.GeneratePeers(2) @@ -350,8 +341,6 @@ func TestPWMSendCancels(t *testing.T) { } func TestStats(t *testing.T) { - test.Flaky(t) - g := &gauge{} wbg := &gauge{} pwm := newPeerWantManager(g, wbg) @@ -453,8 +442,6 @@ func TestStats(t *testing.T) { } func TestStatsOverlappingWantBlockWantHave(t *testing.T) { - test.Flaky(t) - g := &gauge{} wbg := &gauge{} pwm := newPeerWantManager(g, wbg) @@ -494,8 +481,6 @@ func TestStatsOverlappingWantBlockWantHave(t *testing.T) { } func TestStatsRemovePeerOverlappingWantBlockWantHave(t *testing.T) { - test.Flaky(t) - g := &gauge{} wbg := &gauge{} pwm := newPeerWantManager(g, wbg) diff --git a/bitswap/client/internal/providerquerymanager/providerquerymanager_test.go b/bitswap/client/internal/providerquerymanager/providerquerymanager_test.go index 04b309286..afdd19595 100644 --- a/bitswap/client/internal/providerquerymanager/providerquerymanager_test.go +++ b/bitswap/client/internal/providerquerymanager/providerquerymanager_test.go @@ -9,7 +9,6 @@ import ( "time" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" cid "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" ) @@ -59,8 +58,6 @@ func (fpn *fakeProviderNetwork) FindProvidersAsync(ctx context.Context, k cid.Ci } func TestNormalSimultaneousFetch(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(10) fpn := &fakeProviderNetwork{ peersFound: peers, @@ -98,8 +95,6 @@ func TestNormalSimultaneousFetch(t *testing.T) { } func TestDedupingProviderRequests(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(10) fpn := &fakeProviderNetwork{ peersFound: peers, @@ -140,8 +135,6 @@ func TestDedupingProviderRequests(t *testing.T) { } func TestCancelOneRequestDoesNotTerminateAnother(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(10) fpn := &fakeProviderNetwork{ peersFound: peers, @@ -186,8 +179,6 @@ func TestCancelOneRequestDoesNotTerminateAnother(t *testing.T) { } func TestCancelManagerExitsGracefully(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(10) fpn := &fakeProviderNetwork{ peersFound: peers, @@ -223,8 +214,6 @@ func TestCancelManagerExitsGracefully(t *testing.T) { } func TestPeersWithConnectionErrorsNotAddedToPeerList(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(10) fpn := &fakeProviderNetwork{ peersFound: peers, @@ -258,8 +247,6 @@ func TestPeersWithConnectionErrorsNotAddedToPeerList(t *testing.T) { } func TestRateLimitingRequests(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(10) fpn := &fakeProviderNetwork{ peersFound: peers, @@ -299,8 +286,6 @@ func TestRateLimitingRequests(t *testing.T) { } func TestFindProviderTimeout(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(10) fpn := &fakeProviderNetwork{ peersFound: peers, @@ -325,8 +310,6 @@ func TestFindProviderTimeout(t *testing.T) { } func TestFindProviderPreCanceled(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(10) fpn := &fakeProviderNetwork{ peersFound: peers, @@ -352,8 +335,6 @@ func TestFindProviderPreCanceled(t *testing.T) { } func TestCancelFindProvidersAfterCompletion(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(2) fpn := &fakeProviderNetwork{ peersFound: peers, diff --git a/bitswap/client/internal/session/session_test.go b/bitswap/client/internal/session/session_test.go index cf6de1e5a..2eb166f90 100644 --- a/bitswap/client/internal/session/session_test.go +++ b/bitswap/client/internal/session/session_test.go @@ -245,8 +245,6 @@ func TestSessionGetBlocks(t *testing.T) { } func TestSessionFindMorePeers(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), 900*time.Millisecond) defer cancel() fpm := newFakePeerManager() @@ -321,8 +319,6 @@ func TestSessionFindMorePeers(t *testing.T) { } func TestSessionOnPeersExhausted(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) defer cancel() fpm := newFakePeerManager() @@ -485,8 +481,6 @@ func TestSessionFailingToGetFirstBlock(t *testing.T) { } func TestSessionCtxCancelClosesGetBlocksChannel(t *testing.T) { - test.Flaky(t) - fpm := newFakePeerManager() fspm := newFakeSessionPeerManager() fpf := newFakeProviderFinder() @@ -537,8 +531,6 @@ func TestSessionCtxCancelClosesGetBlocksChannel(t *testing.T) { } func TestSessionOnShutdownCalled(t *testing.T) { - test.Flaky(t) - fpm := newFakePeerManager() fspm := newFakeSessionPeerManager() fpf := newFakeProviderFinder() @@ -566,8 +558,6 @@ func TestSessionOnShutdownCalled(t *testing.T) { } func TestSessionReceiveMessageAfterCtxCancel(t *testing.T) { - test.Flaky(t) - ctx, cancelCtx := context.WithTimeout(context.Background(), 20*time.Millisecond) fpm := newFakePeerManager() fspm := newFakeSessionPeerManager() diff --git a/bitswap/client/internal/session/sessionwants_test.go b/bitswap/client/internal/session/sessionwants_test.go index bdb73ebd1..f940ac14f 100644 --- a/bitswap/client/internal/session/sessionwants_test.go +++ b/bitswap/client/internal/session/sessionwants_test.go @@ -4,13 +4,10 @@ import ( "testing" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" cid "github.com/ipfs/go-cid" ) func TestEmptySessionWants(t *testing.T) { - test.Flaky(t) - sw := newSessionWants(broadcastLiveWantsLimit) // Expect these functions to return nothing on a new sessionWants @@ -32,8 +29,6 @@ func TestEmptySessionWants(t *testing.T) { } func TestSessionWants(t *testing.T) { - test.Flaky(t) - sw := newSessionWants(5) cids := testutil.GenerateCids(10) others := testutil.GenerateCids(1) @@ -115,8 +110,6 @@ func TestSessionWants(t *testing.T) { } func TestPrepareBroadcast(t *testing.T) { - test.Flaky(t) - sw := newSessionWants(3) cids := testutil.GenerateCids(10) @@ -177,8 +170,6 @@ func TestPrepareBroadcast(t *testing.T) { // Test that even after GC broadcast returns correct wants func TestPrepareBroadcastAfterGC(t *testing.T) { - test.Flaky(t) - sw := newSessionWants(5) cids := testutil.GenerateCids(liveWantsOrderGCLimit * 2) diff --git a/bitswap/client/internal/session/sessionwantsender_test.go b/bitswap/client/internal/session/sessionwantsender_test.go index 476b13991..86a930f61 100644 --- a/bitswap/client/internal/session/sessionwantsender_test.go +++ b/bitswap/client/internal/session/sessionwantsender_test.go @@ -10,7 +10,6 @@ import ( bspm "github.com/ipfs/boxo/bitswap/client/internal/peermanager" bsspm "github.com/ipfs/boxo/bitswap/client/internal/sessionpeermanager" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" cid "github.com/ipfs/go-cid" peer "github.com/libp2p/go-libp2p/core/peer" ) @@ -142,8 +141,6 @@ func (ep *exhaustedPeers) exhausted() []cid.Cid { } func TestSendWants(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(4) peers := testutil.GeneratePeers(1) peerA := peers[0] @@ -183,8 +180,6 @@ func TestSendWants(t *testing.T) { } func TestSendsWantBlockToOnePeerOnly(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(4) peers := testutil.GeneratePeers(2) peerA := peers[0] @@ -245,8 +240,6 @@ func TestSendsWantBlockToOnePeerOnly(t *testing.T) { } func TestReceiveBlock(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(2) peers := testutil.GeneratePeers(2) peerA := peers[0] @@ -309,8 +302,6 @@ func TestReceiveBlock(t *testing.T) { } func TestCancelWants(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(4) sid := uint64(1) pm := newMockPeerManager() @@ -345,8 +336,6 @@ func TestCancelWants(t *testing.T) { } func TestRegisterSessionWithPeerManager(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(2) peers := testutil.GeneratePeers(2) peerA := peers[0] @@ -387,8 +376,6 @@ func TestRegisterSessionWithPeerManager(t *testing.T) { } func TestProtectConnFirstPeerToSendWantedBlock(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(2) peers := testutil.GeneratePeers(3) peerA := peers[0] @@ -445,8 +432,6 @@ func TestProtectConnFirstPeerToSendWantedBlock(t *testing.T) { } func TestPeerUnavailable(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(2) peers := testutil.GeneratePeers(2) peerA := peers[0] @@ -514,8 +499,6 @@ func TestPeerUnavailable(t *testing.T) { } func TestPeersExhausted(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(3) peers := testutil.GeneratePeers(2) peerA := peers[0] @@ -593,8 +576,6 @@ func TestPeersExhausted(t *testing.T) { // - the remaining peer becomes unavailable // onPeersExhausted should be sent for that CID func TestPeersExhaustedLastWaitingPeerUnavailable(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(2) peers := testutil.GeneratePeers(2) peerA := peers[0] @@ -644,8 +625,6 @@ func TestPeersExhaustedLastWaitingPeerUnavailable(t *testing.T) { // Tests that when all the peers are removed from the session // onPeersExhausted should be called with all outstanding CIDs func TestPeersExhaustedAllPeersUnavailable(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(3) peers := testutil.GeneratePeers(2) peerA := peers[0] @@ -688,8 +667,6 @@ func TestPeersExhaustedAllPeersUnavailable(t *testing.T) { } func TestConsecutiveDontHaveLimit(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(peerDontHaveLimit + 10) p := testutil.GeneratePeers(1)[0] sid := uint64(1) @@ -748,8 +725,6 @@ func TestConsecutiveDontHaveLimit(t *testing.T) { } func TestConsecutiveDontHaveLimitInterrupted(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(peerDontHaveLimit + 10) p := testutil.GeneratePeers(1)[0] sid := uint64(1) @@ -807,8 +782,6 @@ func TestConsecutiveDontHaveLimitInterrupted(t *testing.T) { } func TestConsecutiveDontHaveReinstateAfterRemoval(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(peerDontHaveLimit + 10) p := testutil.GeneratePeers(1)[0] sid := uint64(1) @@ -895,8 +868,6 @@ func TestConsecutiveDontHaveReinstateAfterRemoval(t *testing.T) { } func TestConsecutiveDontHaveDontRemoveIfHasWantedBlock(t *testing.T) { - test.Flaky(t) - cids := testutil.GenerateCids(peerDontHaveLimit + 10) p := testutil.GeneratePeers(1)[0] sid := uint64(1) diff --git a/bitswap/client/internal/sessioninterestmanager/sessioninterestmanager_test.go b/bitswap/client/internal/sessioninterestmanager/sessioninterestmanager_test.go index 85857b9ba..f2b4d8aa0 100644 --- a/bitswap/client/internal/sessioninterestmanager/sessioninterestmanager_test.go +++ b/bitswap/client/internal/sessioninterestmanager/sessioninterestmanager_test.go @@ -4,13 +4,10 @@ import ( "testing" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" cid "github.com/ipfs/go-cid" ) func TestEmpty(t *testing.T) { - test.Flaky(t) - sim := New() ses := uint64(1) @@ -25,8 +22,6 @@ func TestEmpty(t *testing.T) { } func TestBasic(t *testing.T) { - test.Flaky(t) - sim := New() ses1 := uint64(1) @@ -62,8 +57,6 @@ func TestBasic(t *testing.T) { } func TestInterestedSessions(t *testing.T) { - test.Flaky(t) - sim := New() ses := uint64(1) @@ -91,8 +84,6 @@ func TestInterestedSessions(t *testing.T) { } func TestRemoveSession(t *testing.T) { - test.Flaky(t) - sim := New() ses1 := uint64(1) @@ -121,8 +112,6 @@ func TestRemoveSession(t *testing.T) { } func TestRemoveSessionInterested(t *testing.T) { - test.Flaky(t) - sim := New() ses1 := uint64(1) @@ -159,8 +148,6 @@ func TestRemoveSessionInterested(t *testing.T) { } func TestSplitWantedUnwanted(t *testing.T) { - test.Flaky(t) - blks := testutil.GenerateBlocksOfSize(3, 1024) sim := New() ses1 := uint64(1) diff --git a/bitswap/client/internal/sessionmanager/sessionmanager_test.go b/bitswap/client/internal/sessionmanager/sessionmanager_test.go index 5ecabfdb3..86f7f8df5 100644 --- a/bitswap/client/internal/sessionmanager/sessionmanager_test.go +++ b/bitswap/client/internal/sessionmanager/sessionmanager_test.go @@ -13,7 +13,6 @@ import ( bssession "github.com/ipfs/boxo/bitswap/client/internal/session" bssim "github.com/ipfs/boxo/bitswap/client/internal/sessioninterestmanager" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" delay "github.com/ipfs/go-ipfs-delay" @@ -113,8 +112,6 @@ func peerManagerFactory(ctx context.Context, id uint64) bssession.SessionPeerMan } func TestReceiveFrom(t *testing.T) { - test.Flaky(t) - ctx := context.Background() ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -162,8 +159,6 @@ func TestReceiveFrom(t *testing.T) { } func TestReceiveBlocksWhenManagerShutdown(t *testing.T) { - test.Flaky(t) - ctx := context.Background() ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -199,8 +194,6 @@ func TestReceiveBlocksWhenManagerShutdown(t *testing.T) { } func TestReceiveBlocksWhenSessionContextCancelled(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() notif := notifications.New() @@ -236,8 +229,6 @@ func TestReceiveBlocksWhenSessionContextCancelled(t *testing.T) { } func TestShutdown(t *testing.T) { - test.Flaky(t) - ctx := context.Background() ctx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/bitswap/client/internal/sessionpeermanager/sessionpeermanager_test.go b/bitswap/client/internal/sessionpeermanager/sessionpeermanager_test.go index fc1d7274d..0d9275579 100644 --- a/bitswap/client/internal/sessionpeermanager/sessionpeermanager_test.go +++ b/bitswap/client/internal/sessionpeermanager/sessionpeermanager_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" peer "github.com/libp2p/go-libp2p/core/peer" ) @@ -79,8 +78,6 @@ func (fpt *fakePeerTagger) isProtected(p peer.ID) bool { } func TestAddPeers(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(2) spm := New(1, &fakePeerTagger{}) @@ -101,8 +98,6 @@ func TestAddPeers(t *testing.T) { } func TestRemovePeers(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(2) spm := New(1, &fakePeerTagger{}) @@ -129,8 +124,6 @@ func TestRemovePeers(t *testing.T) { } func TestHasPeers(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(2) spm := New(1, &fakePeerTagger{}) @@ -160,8 +153,6 @@ func TestHasPeers(t *testing.T) { } func TestHasPeer(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(2) spm := New(1, &fakePeerTagger{}) @@ -190,8 +181,6 @@ func TestHasPeer(t *testing.T) { } func TestPeers(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(2) spm := New(1, &fakePeerTagger{}) @@ -216,8 +205,6 @@ func TestPeers(t *testing.T) { } func TestPeersDiscovered(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(2) spm := New(1, &fakePeerTagger{}) @@ -237,8 +224,6 @@ func TestPeersDiscovered(t *testing.T) { } func TestPeerTagging(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(2) fpt := &fakePeerTagger{} spm := New(1, fpt) @@ -265,8 +250,6 @@ func TestPeerTagging(t *testing.T) { } func TestProtectConnection(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(1) peerA := peers[0] fpt := newFakePeerTagger() @@ -293,8 +276,6 @@ func TestProtectConnection(t *testing.T) { } func TestShutdown(t *testing.T) { - test.Flaky(t) - peers := testutil.GeneratePeers(2) fpt := newFakePeerTagger() spm := New(1, fpt) diff --git a/bitswap/client/wantlist/wantlist_test.go b/bitswap/client/wantlist/wantlist_test.go index 035786ea3..07d4ce415 100644 --- a/bitswap/client/wantlist/wantlist_test.go +++ b/bitswap/client/wantlist/wantlist_test.go @@ -4,7 +4,6 @@ import ( "testing" pb "github.com/ipfs/boxo/bitswap/message/pb" - "github.com/ipfs/boxo/internal/test" cid "github.com/ipfs/go-cid" "github.com/stretchr/testify/require" ) @@ -41,8 +40,6 @@ func assertHasCid(t *testing.T, w wli, c cid.Cid) { } func TestBasicWantlist(t *testing.T) { - test.Flaky(t) - wl := New() if !wl.Add(testcids[0], 5, pb.Message_Wantlist_Block) { @@ -80,8 +77,6 @@ func TestBasicWantlist(t *testing.T) { } func TestAddHaveThenBlock(t *testing.T) { - test.Flaky(t) - wl := New() wl.Add(testcids[0], 5, pb.Message_Wantlist_Have) @@ -97,8 +92,6 @@ func TestAddHaveThenBlock(t *testing.T) { } func TestAddBlockThenHave(t *testing.T) { - test.Flaky(t) - wl := New() wl.Add(testcids[0], 5, pb.Message_Wantlist_Block) @@ -114,8 +107,6 @@ func TestAddBlockThenHave(t *testing.T) { } func TestAddHaveThenRemoveBlock(t *testing.T) { - test.Flaky(t) - wl := New() wl.Add(testcids[0], 5, pb.Message_Wantlist_Have) @@ -128,8 +119,6 @@ func TestAddHaveThenRemoveBlock(t *testing.T) { } func TestAddBlockThenRemoveHave(t *testing.T) { - test.Flaky(t) - wl := New() wl.Add(testcids[0], 5, pb.Message_Wantlist_Block) @@ -145,8 +134,6 @@ func TestAddBlockThenRemoveHave(t *testing.T) { } func TestAddHaveThenRemoveAny(t *testing.T) { - test.Flaky(t) - wl := New() wl.Add(testcids[0], 5, pb.Message_Wantlist_Have) @@ -159,8 +146,6 @@ func TestAddHaveThenRemoveAny(t *testing.T) { } func TestAddBlockThenRemoveAny(t *testing.T) { - test.Flaky(t) - wl := New() wl.Add(testcids[0], 5, pb.Message_Wantlist_Block) @@ -173,8 +158,6 @@ func TestAddBlockThenRemoveAny(t *testing.T) { } func TestAbsort(t *testing.T) { - test.Flaky(t) - wl := New() wl.Add(testcids[0], 5, pb.Message_Wantlist_Block) wl.Add(testcids[1], 4, pb.Message_Wantlist_Have) @@ -221,8 +204,6 @@ func TestAbsort(t *testing.T) { } func TestSortEntries(t *testing.T) { - test.Flaky(t) - wl := New() wl.Add(testcids[0], 3, pb.Message_Wantlist_Block) @@ -239,8 +220,6 @@ func TestSortEntries(t *testing.T) { // Test adding and removing interleaved with checking entries to make sure we clear the cache. func TestCache(t *testing.T) { - test.Flaky(t) - wl := New() wl.Add(testcids[0], 3, pb.Message_Wantlist_Block) diff --git a/bitswap/network/connecteventmanager_test.go b/bitswap/network/connecteventmanager_test.go index e3904ee55..bb3c52266 100644 --- a/bitswap/network/connecteventmanager_test.go +++ b/bitswap/network/connecteventmanager_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" ) @@ -46,8 +45,6 @@ func wait(t *testing.T, c *connectEventManager) { } func TestConnectEventManagerConnectDisconnect(t *testing.T) { - test.Flaky(t) - connListener := newMockConnListener() peers := testutil.GeneratePeers(2) cem := newConnectEventManager(connListener) @@ -87,8 +84,6 @@ func TestConnectEventManagerConnectDisconnect(t *testing.T) { } func TestConnectEventManagerMarkUnresponsive(t *testing.T) { - test.Flaky(t) - connListener := newMockConnListener() p := testutil.GeneratePeers(1)[0] cem := newConnectEventManager(connListener) @@ -138,8 +133,6 @@ func TestConnectEventManagerMarkUnresponsive(t *testing.T) { } func TestConnectEventManagerDisconnectAfterMarkUnresponsive(t *testing.T) { - test.Flaky(t) - connListener := newMockConnListener() p := testutil.GeneratePeers(1)[0] cem := newConnectEventManager(connListener) diff --git a/bitswap/network/ipfs_impl_test.go b/bitswap/network/ipfs_impl_test.go index 175957799..196fc7d55 100644 --- a/bitswap/network/ipfs_impl_test.go +++ b/bitswap/network/ipfs_impl_test.go @@ -12,7 +12,6 @@ import ( bsnet "github.com/ipfs/boxo/bitswap/network" "github.com/ipfs/boxo/bitswap/network/internal" tn "github.com/ipfs/boxo/bitswap/testnet" - "github.com/ipfs/boxo/internal/test" mockrouting "github.com/ipfs/boxo/routing/mock" ds "github.com/ipfs/go-datastore" blocksutil "github.com/ipfs/go-ipfs-blocksutil" @@ -164,8 +163,6 @@ func (eh *ErrHost) setTimeoutState(timingOut bool) { } func TestMessageSendAndReceive(t *testing.T) { - test.Flaky(t) - // create network ctx := context.Background() ctx, cancel := context.WithTimeout(ctx, 10*time.Second) @@ -334,8 +331,6 @@ func prepareNetwork(t *testing.T, ctx context.Context, p1 tnet.Identity, r1 *rec } func TestMessageResendAfterError(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -382,8 +377,6 @@ func TestMessageResendAfterError(t *testing.T) { } func TestMessageSendTimeout(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -425,8 +418,6 @@ func TestMessageSendTimeout(t *testing.T) { } func TestMessageSendNotSupportedResponse(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -459,8 +450,6 @@ func TestMessageSendNotSupportedResponse(t *testing.T) { } func TestSupportsHave(t *testing.T) { - test.Flaky(t) - ctx := context.Background() mn := mocknet.New() defer mn.Close() @@ -675,8 +664,6 @@ func testNetworkCounters(t *testing.T, n1 int, n2 int) { } func TestNetworkCounters(t *testing.T) { - test.Flaky(t) - for n := 0; n < 11; n++ { testNetworkCounters(t, 10-n, n) } diff --git a/bitswap/network/ipfs_impl_timeout_test.go b/bitswap/network/ipfs_impl_timeout_test.go index 178c2fb69..fdbe8e950 100644 --- a/bitswap/network/ipfs_impl_timeout_test.go +++ b/bitswap/network/ipfs_impl_timeout_test.go @@ -4,13 +4,10 @@ import ( "testing" "time" - "github.com/ipfs/boxo/internal/test" "github.com/stretchr/testify/require" ) func TestSendTimeout(t *testing.T) { - test.Flaky(t) - require.Equal(t, minSendTimeout, sendTimeout(0)) require.Equal(t, maxSendTimeout, sendTimeout(1<<30)) diff --git a/bitswap/server/internal/decision/blockstoremanager_test.go b/bitswap/server/internal/decision/blockstoremanager_test.go index 3d4a8ea37..1192873f3 100644 --- a/bitswap/server/internal/decision/blockstoremanager_test.go +++ b/bitswap/server/internal/decision/blockstoremanager_test.go @@ -9,7 +9,6 @@ import ( "github.com/ipfs/boxo/bitswap/internal/testutil" blockstore "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/boxo/internal/test" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -34,8 +33,6 @@ func newBlockstoreManagerForTesting( } func TestBlockstoreManagerNotFoundKey(t *testing.T) { - test.Flaky(t) - ctx := context.Background() bsdelay := delay.Fixed(3 * time.Millisecond) dstore := ds_sync.MutexWrap(delayed.New(ds.NewMapDatastore(), bsdelay)) @@ -74,8 +71,6 @@ func TestBlockstoreManagerNotFoundKey(t *testing.T) { } func TestBlockstoreManager(t *testing.T) { - test.Flaky(t) - ctx := context.Background() bsdelay := delay.Fixed(3 * time.Millisecond) dstore := ds_sync.MutexWrap(delayed.New(ds.NewMapDatastore(), bsdelay)) @@ -158,8 +153,6 @@ func TestBlockstoreManager(t *testing.T) { } func TestBlockstoreManagerConcurrency(t *testing.T) { - test.Flaky(t) - ctx := context.Background() bsdelay := delay.Fixed(3 * time.Millisecond) dstore := ds_sync.MutexWrap(delayed.New(ds.NewMapDatastore(), bsdelay)) @@ -201,8 +194,6 @@ func TestBlockstoreManagerConcurrency(t *testing.T) { } func TestBlockstoreManagerClose(t *testing.T) { - test.Flaky(t) - ctx := context.Background() delayTime := 20 * time.Millisecond bsdelay := delay.Fixed(delayTime) @@ -224,8 +215,6 @@ func TestBlockstoreManagerClose(t *testing.T) { bsm.stop() - time.Sleep(5 * time.Millisecond) - before := time.Now() _, err = bsm.getBlockSizes(ctx, ks) if err == nil { @@ -238,8 +227,6 @@ func TestBlockstoreManagerClose(t *testing.T) { } func TestBlockstoreManagerCtxDone(t *testing.T) { - test.Flaky(t) - delayTime := 20 * time.Millisecond bsdelay := delay.Fixed(delayTime) diff --git a/bitswap/server/internal/decision/engine_test.go b/bitswap/server/internal/decision/engine_test.go index 922836042..efc4408ff 100644 --- a/bitswap/server/internal/decision/engine_test.go +++ b/bitswap/server/internal/decision/engine_test.go @@ -17,7 +17,6 @@ import ( message "github.com/ipfs/boxo/bitswap/message" pb "github.com/ipfs/boxo/bitswap/message/pb" blockstore "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/boxo/internal/test" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -112,8 +111,6 @@ func newTestEngineWithSampling(ctx context.Context, idStr string, peerSampleInte } func TestConsistentAccounting(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() sender := newTestEngine(ctx, "Ernie") @@ -149,8 +146,6 @@ func TestConsistentAccounting(t *testing.T) { } func TestPeerIsAddedToPeersWhenMessageSent(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() sanfrancisco := newTestEngine(ctx, "sf") @@ -205,8 +200,6 @@ func newEngineForTesting( } func TestOutboxClosedWhenEngineClosed(t *testing.T) { - test.Flaky(t) - t.SkipNow() // TODO implement *Engine.Close ctx := context.Background() e := newEngineForTesting(ctx, blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())), &fakePeerTagger{}, "localhost", 0, WithScoreLedger(NewTestScoreLedger(shortTerm, nil, clock.New())), WithBlockstoreWorkerCount(4)) @@ -227,8 +220,6 @@ func TestOutboxClosedWhenEngineClosed(t *testing.T) { } func TestPartnerWantHaveWantBlockNonActive(t *testing.T) { - test.Flaky(t) - alphabet := "abcdefghijklmnopqrstuvwxyz" vowels := "aeiou" @@ -569,8 +560,6 @@ func TestPartnerWantHaveWantBlockNonActive(t *testing.T) { } func TestPartnerWantHaveWantBlockActive(t *testing.T) { - test.Flaky(t) - alphabet := "abcdefghijklmnopqrstuvwxyz" bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) @@ -843,8 +832,6 @@ func formatPresencesDiff(presences []message.BlockPresence, expHaves []string, e } func TestPartnerWantsThenCancels(t *testing.T) { - test.Flaky(t) - numRounds := 10 if testing.Short() { numRounds = 1 @@ -907,8 +894,6 @@ func TestPartnerWantsThenCancels(t *testing.T) { } func TestSendReceivedBlocksToPeersThatWantThem(t *testing.T) { - test.Flaky(t) - bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) partner := libp2ptest.RandPeerIDFatal(t) otherPeer := libp2ptest.RandPeerIDFatal(t) @@ -955,8 +940,6 @@ func TestSendReceivedBlocksToPeersThatWantThem(t *testing.T) { } func TestSendDontHave(t *testing.T) { - test.Flaky(t) - bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) partner := libp2ptest.RandPeerIDFatal(t) otherPeer := libp2ptest.RandPeerIDFatal(t) @@ -1023,8 +1006,6 @@ func TestSendDontHave(t *testing.T) { } func TestWantlistForPeer(t *testing.T) { - test.Flaky(t) - bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) partner := libp2ptest.RandPeerIDFatal(t) otherPeer := libp2ptest.RandPeerIDFatal(t) @@ -1062,8 +1043,6 @@ func TestWantlistForPeer(t *testing.T) { } func TestTaskComparator(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() @@ -1118,8 +1097,6 @@ func TestTaskComparator(t *testing.T) { } func TestPeerBlockFilter(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() @@ -1279,8 +1256,6 @@ func TestPeerBlockFilter(t *testing.T) { } func TestPeerBlockFilterMutability(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() @@ -1450,8 +1425,6 @@ func TestPeerBlockFilterMutability(t *testing.T) { } func TestTaggingPeers(t *testing.T) { - test.Flaky(t) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() sanfrancisco := newTestEngine(ctx, "sf") @@ -1480,8 +1453,6 @@ func TestTaggingPeers(t *testing.T) { } func TestTaggingUseful(t *testing.T) { - test.Flaky(t) - peerSampleIntervalHalf := 10 * time.Millisecond ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/bitswap/server/internal/decision/taskmerger_test.go b/bitswap/server/internal/decision/taskmerger_test.go index e0ce46ed6..ae3b0384d 100644 --- a/bitswap/server/internal/decision/taskmerger_test.go +++ b/bitswap/server/internal/decision/taskmerger_test.go @@ -4,14 +4,11 @@ import ( "testing" "github.com/ipfs/boxo/bitswap/internal/testutil" - "github.com/ipfs/boxo/internal/test" "github.com/ipfs/go-peertaskqueue" "github.com/ipfs/go-peertaskqueue/peertask" ) func TestPushHaveVsBlock(t *testing.T) { - test.Flaky(t) - partner := testutil.GeneratePeers(1)[0] wantHave := peertask.Task{ @@ -64,8 +61,6 @@ func TestPushHaveVsBlock(t *testing.T) { } func TestPushSizeInfo(t *testing.T) { - test.Flaky(t) - partner := testutil.GeneratePeers(1)[0] wantBlockBlockSize := 10 @@ -178,8 +173,6 @@ func TestPushSizeInfo(t *testing.T) { } func TestPushHaveVsBlockActive(t *testing.T) { - test.Flaky(t) - partner := testutil.GeneratePeers(1)[0] wantBlock := peertask.Task{ @@ -234,8 +227,6 @@ func TestPushHaveVsBlockActive(t *testing.T) { } func TestPushSizeInfoActive(t *testing.T) { - test.Flaky(t) - partner := testutil.GeneratePeers(1)[0] wantBlock := peertask.Task{ diff --git a/blockservice/test/mock.go b/blockservice/test/mock.go index fa6469fb6..e32b10b99 100644 --- a/blockservice/test/mock.go +++ b/blockservice/test/mock.go @@ -9,7 +9,7 @@ import ( ) // Mocks returns |n| connected mock Blockservices -func Mocks(n int) []blockservice.BlockService { +func Mocks(n int, opts ...blockservice.Option) []blockservice.BlockService { net := tn.VirtualNetwork(mockrouting.NewServer(), delay.Fixed(0)) sg := testinstance.NewTestInstanceGenerator(net, nil, nil) @@ -17,7 +17,7 @@ func Mocks(n int) []blockservice.BlockService { var servs []blockservice.BlockService for _, i := range instances { - servs = append(servs, blockservice.New(i.Blockstore(), i.Exchange)) + servs = append(servs, blockservice.New(i.Blockstore(), i.Exchange, opts...)) } return servs } diff --git a/provider/reprovider_test.go b/provider/reprovider_test.go index bfb8fc187..4efd9bda1 100644 --- a/provider/reprovider_test.go +++ b/provider/reprovider_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/ipfs/boxo/internal/test" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" @@ -180,6 +181,9 @@ func testProvider(t *testing.T, singleProvide bool) { } func TestOfflineRecordsThenOnlineRepublish(t *testing.T) { + if runtime.GOOS == "windows" { + test.Flaky(t) + } // Don't run in Parallel as this test is time sensitive. someHash, err := mh.Sum([]byte("Vires in Numeris!"), mh.BLAKE3, -1) diff --git a/routing/http/README.md b/routing/http/README.md index 65650ed50..0f0281f8f 100644 --- a/routing/http/README.md +++ b/routing/http/README.md @@ -1,24 +1,9 @@ -go-delegated-routing +Routing V1 Server and Client ======================= -> Delegated routing Client and Server over Reframe RPC - -This package provides delegated routing implementation in Go: -- Client (for IPFS nodes like [Kubo](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingrouters-parameters)), -- Server (for public indexers such as https://cid.contact) +> Delegated Routing V1 Server and Client over HTTP API. ## Documentation -- Go docs: https://pkg.go.dev/github.com/ipfs/boxo/routing/http/ - -## Lead Maintainer - -🦗🎶 - -## Contributing - -Contributions are welcome! This repository is part of the IPFS project and therefore governed by our [contributing guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md). - -## License - -[SPDX-License-Identifier: Apache-2.0 OR MIT](LICENSE.md) \ No newline at end of file +- Go Documentation: https://pkg.go.dev/github.com/ipfs/boxo/routing/http +- Routing V1 Specification: https://specs.ipfs.tech/routing/http-routing-v1/ diff --git a/routing/http/client/client.go b/routing/http/client/client.go index c504a0315..4a0d29b33 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -16,14 +16,12 @@ import ( ipns "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/routing/http/contentrouter" "github.com/ipfs/boxo/routing/http/internal/drjson" - "github.com/ipfs/boxo/routing/http/server" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" jsontypes "github.com/ipfs/boxo/routing/http/types/json" "github.com/ipfs/boxo/routing/http/types/ndjson" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - record "github.com/libp2p/go-libp2p-record" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" @@ -31,7 +29,7 @@ import ( var ( _ contentrouter.Client = &client{} - logger = logging.Logger("service/delegatedrouting") + logger = logging.Logger("routing/http/client") defaultHTTPClient = &http.Client{ Transport: &ResponseBodyLimitedTransport{ RoundTripper: http.DefaultTransport, @@ -50,18 +48,17 @@ const ( type client struct { baseURL string httpClient httpClient - validator record.Validator clock clock.Clock - - accepts string + accepts string peerID peer.ID addrs []types.Multiaddr identity crypto.PrivKey - // called immeidately after signing a provide req - // used for testing, e.g. testing the server with a mangled signature - afterSignCallback func(req *types.WriteBitswapProviderRecord) + // Called immediately after signing a provide request. It is used + // for testing, e.g., testing the server with a mangled signature. + //lint:ignore SA1019 // ignore staticcheck + afterSignCallback func(req *types.WriteBitswapRecord) } // defaultUserAgent is used as a fallback to inform HTTP server which library @@ -121,12 +118,11 @@ func WithStreamResultsRequired() Option { } // New creates a content routing API client. -// The Provider and identity parameters are option. If they are nil, the `Provide` method will not function. +// The Provider and identity parameters are option. If they are nil, the [client.ProvideBitswap] method will not function. func New(baseURL string, opts ...Option) (*client, error) { client := &client{ baseURL: baseURL, httpClient: defaultHTTPClient, - validator: ipns.Validator{}, clock: clock.New(), accepts: strings.Join([]string{mediaTypeNDJSON, mediaTypeJSON}, ","), } @@ -164,11 +160,11 @@ func (c *measuringIter[T]) Close() error { return c.Iter.Close() } -func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.ResultIter[types.ProviderResponse], err error) { +func (c *client) FindProviders(ctx context.Context, key cid.Cid) (providers iter.ResultIter[types.Record], err error) { // TODO test measurements m := newMeasurement("FindProviders") - url := c.baseURL + server.ProvidePath + key.String() + url := c.baseURL + "/routing/v1/providers/" + key.String() req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err @@ -192,7 +188,7 @@ func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.Res if resp.StatusCode == http.StatusNotFound { resp.Body.Close() m.record(ctx) - return iter.FromSlice[iter.Result[types.ProviderResponse]](nil), nil + return iter.FromSlice[iter.Result[types.Record]](nil), nil } if resp.StatusCode != http.StatusOK { @@ -220,24 +216,27 @@ func (c *client) FindProviders(ctx context.Context, key cid.Cid) (provs iter.Res } }() - var it iter.ResultIter[types.ProviderResponse] + var it iter.ResultIter[types.Record] switch mediaType { case mediaTypeJSON: - parsedResp := &jsontypes.ReadProvidersResponse{} + parsedResp := &jsontypes.ProvidersResponse{} err = json.NewDecoder(resp.Body).Decode(parsedResp) - var sliceIt iter.Iter[types.ProviderResponse] = iter.FromSlice(parsedResp.Providers) + var sliceIt iter.Iter[types.Record] = iter.FromSlice(parsedResp.Providers) it = iter.ToResultIter(sliceIt) case mediaTypeNDJSON: skipBodyClose = true - it = ndjson.NewReadProvidersResponseIter(resp.Body) + it = ndjson.NewRecordsIter(resp.Body) default: logger.Errorw("unknown media type", "MediaType", mediaType, "ContentType", respContentType) return nil, errors.New("unknown content type") } - return &measuringIter[iter.Result[types.ProviderResponse]]{Iter: it, ctx: ctx, m: m}, nil + return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil } +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 func (c *client) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) { if c.identity == nil { return 0, errors.New("cannot provide Bitswap records without an identity") @@ -253,7 +252,7 @@ func (c *client) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Du now := c.clock.Now() - req := types.WriteBitswapProviderRecord{ + req := types.WriteBitswapRecord{ Protocol: "transport-bitswap", Schema: types.SchemaBitswap, Payload: types.BitswapPayload{ @@ -282,10 +281,13 @@ func (c *client) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Du } // ProvideAsync makes a provide request to a delegated router -func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.WriteBitswapProviderRecord) (time.Duration, error) { - req := jsontypes.WriteProvidersRequest{Providers: []types.WriteProviderRecord{bswp}} +// +//lint:ignore SA1019 // ignore staticcheck +func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.WriteBitswapRecord) (time.Duration, error) { + //lint:ignore SA1019 // ignore staticcheck + req := jsontypes.WriteProvidersRequest{Providers: []types.Record{bswp}} - url := c.baseURL + server.ProvidePath + url := c.baseURL + "/routing/v1/providers/" b, err := drjson.MarshalJSONBytes(req) if err != nil { @@ -306,6 +308,8 @@ func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.Wri if resp.StatusCode != http.StatusOK { return 0, httpError(resp.StatusCode, resp.Body) } + + //lint:ignore SA1019 // ignore staticcheck var provideResult jsontypes.WriteProvidersResponse err = json.NewDecoder(resp.Body).Decode(&provideResult) if err != nil { @@ -315,7 +319,8 @@ func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.Wri return 0, fmt.Errorf("expected 1 result but got %d", len(provideResult.ProvideResults)) } - v, ok := provideResult.ProvideResults[0].(*types.WriteBitswapProviderRecordResponse) + //lint:ignore SA1019 // ignore staticcheck + v, ok := provideResult.ProvideResults[0].(*types.WriteBitswapRecordResponse) if !ok { return 0, fmt.Errorf("expected AdvisoryTTL field") } @@ -327,7 +332,80 @@ func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.Wri return 0, nil } -func (c *client) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (c *client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) { + m := newMeasurement("FindPeers") + + url := c.baseURL + "/routing/v1/peers/" + peer.ToCid(pid).String() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", c.accepts) + + m.host = req.Host + + start := c.clock.Now() + resp, err := c.httpClient.Do(req) + + m.err = err + m.latency = c.clock.Since(start) + + if err != nil { + m.record(ctx) + return nil, err + } + + m.statusCode = resp.StatusCode + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + m.record(ctx) + return iter.FromSlice[iter.Result[types.Record]](nil), nil + } + + if resp.StatusCode != http.StatusOK { + err := httpError(resp.StatusCode, resp.Body) + resp.Body.Close() + m.record(ctx) + return nil, err + } + + respContentType := resp.Header.Get("Content-Type") + mediaType, _, err := mime.ParseMediaType(respContentType) + if err != nil { + resp.Body.Close() + m.err = err + m.record(ctx) + return nil, fmt.Errorf("parsing Content-Type: %w", err) + } + + m.mediaType = mediaType + + var skipBodyClose bool + defer func() { + if !skipBodyClose { + resp.Body.Close() + } + }() + + var it iter.ResultIter[types.Record] + switch mediaType { + case mediaTypeJSON: + parsedResp := &jsontypes.PeersResponse{} + err = json.NewDecoder(resp.Body).Decode(parsedResp) + var sliceIt iter.Iter[types.Record] = iter.FromSlice(parsedResp.Peers) + it = iter.ToResultIter(sliceIt) + case mediaTypeNDJSON: + skipBodyClose = true + it = ndjson.NewRecordsIter(resp.Body) + default: + logger.Errorw("unknown media type", "MediaType", mediaType, "ContentType", respContentType) + return nil, errors.New("unknown content type") + } + + return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil +} + +func (c *client) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { url := c.baseURL + "/routing/v1/ipns/" + name.String() httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -365,7 +443,7 @@ func (c *client) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Reco return record, nil } -func (c *client) ProvideIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (c *client) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { url := c.baseURL + "/routing/v1/ipns/" + name.String() rawRecord, err := ipns.MarshalRecord(record) diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index c1690b3f2..95683bc3f 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -3,6 +3,7 @@ package client import ( "context" "crypto/rand" + "encoding/json" "errors" "net/http" "net/http/httptest" @@ -31,27 +32,28 @@ import ( type mockContentRouter struct{ mock.Mock } -func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) { +func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key, limit) - return args.Get(0).(iter.ResultIter[types.ProviderResponse]), args.Error(1) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } +//lint:ignore SA1019 // ignore staticcheck func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) { args := m.Called(ctx, req) return args.Get(0).(time.Duration), args.Error(1) } -func (m *mockContentRouter) Provide(ctx context.Context, req *server.WriteProvideRequest) (types.ProviderResponse, error) { - args := m.Called(ctx, req) - return args.Get(0).(types.ProviderResponse), args.Error(1) +func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { + args := m.Called(ctx, pid, limit) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockContentRouter) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (m *mockContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } -func (m *mockContentRouter) ProvideIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (m *mockContentRouter) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } @@ -144,6 +146,13 @@ func makeCID() cid.Cid { return c } +func drAddrsToAddrs(drmas []types.Multiaddr) (addrs []multiaddr.Multiaddr) { + for _, a := range drmas { + addrs = append(addrs, a.Multiaddr) + } + return +} + func addrsToDRAddrs(addrs []multiaddr.Multiaddr) (drmas []types.Multiaddr) { for _, a := range addrs { drmas = append(drmas, types.Multiaddr{Multiaddr: a}) @@ -151,19 +160,26 @@ func addrsToDRAddrs(addrs []multiaddr.Multiaddr) (drmas []types.Multiaddr) { return } -func drAddrsToAddrs(drmas []types.Multiaddr) (addrs []multiaddr.Multiaddr) { - for _, a := range drmas { - addrs = append(addrs, a.Multiaddr) +func makePeerRecord() types.PeerRecord { + peerID, addrs, _ := makeProviderAndIdentity() + return types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &peerID, + Protocols: []string{"transport-bitswap"}, + Addrs: addrsToDRAddrs(addrs), + Extra: map[string]json.RawMessage{}, } - return } -func makeBSReadProviderResp() types.ReadBitswapProviderRecord { +//lint:ignore SA1019 // ignore staticcheck +func makeBitswapRecord() types.BitswapRecord { peerID, addrs, _ := makeProviderAndIdentity() - return types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", + //lint:ignore SA1019 // ignore staticcheck + return types.BitswapRecord{ + //lint:ignore SA1019 // ignore staticcheck Schema: types.SchemaBitswap, ID: &peerID, + Protocol: "transport-bitswap", Addrs: addrsToDRAddrs(addrs), } } @@ -208,35 +224,46 @@ func (e *osErrContains) errContains(t *testing.T, err error) { } func TestClient_FindProviders(t *testing.T) { - bsReadProvResp := makeBSReadProviderResp() - bitswapProvs := []iter.Result[types.ProviderResponse]{ - {Val: &bsReadProvResp}, + peerRecord := makePeerRecord() + peerProviders := []iter.Result[types.Record]{ + {Val: &peerRecord}, + } + + bitswapRecord := makeBitswapRecord() + bitswapProviders := []iter.Result[types.Record]{ + {Val: &bitswapRecord}, } cases := []struct { name string httpStatusCode int stopServer bool - routerProvs []iter.Result[types.ProviderResponse] + routerResult []iter.Result[types.Record] routerErr error clientRequiresStreaming bool serverStreamingDisabled bool expErrContains osErrContains - expProvs []iter.Result[types.ProviderResponse] + expResult []iter.Result[types.Record] expStreamingResponse bool expJSONResponse bool }{ { name: "happy case", - routerProvs: bitswapProvs, - expProvs: bitswapProvs, + routerResult: peerProviders, + expResult: peerProviders, + expStreamingResponse: true, + }, + { + name: "happy case (with deprecated bitswap schema)", + routerResult: bitswapProviders, + expResult: bitswapProviders, expStreamingResponse: true, }, { name: "server doesn't support streaming", - routerProvs: bitswapProvs, - expProvs: bitswapProvs, + routerResult: peerProviders, + expResult: peerProviders, serverStreamingDisabled: true, expJSONResponse: true, }, @@ -262,7 +289,7 @@ func TestClient_FindProviders(t *testing.T) { { name: "returns no providers if the HTTP server returns a 404 respones", httpStatusCode: 404, - expProvs: nil, + expResult: nil, }, } for _, c := range cases { @@ -287,6 +314,7 @@ func TestClient_FindProviders(t *testing.T) { assert.Equal(t, mediaTypeNDJSON, r.Header.Get("Content-Type")) }) } + if c.expJSONResponse { onRespReceived = append(onRespReceived, func(r *http.Response) { assert.Equal(t, mediaTypeJSON, r.Header.Get("Content-Type")) @@ -315,20 +343,18 @@ func TestClient_FindProviders(t *testing.T) { } cid := makeCID() - findProvsIter := iter.FromSlice(c.routerProvs) - + routerResultIter := iter.FromSlice(c.routerResult) if c.expStreamingResponse { - router.On("FindProviders", mock.Anything, cid, 0).Return(findProvsIter, c.routerErr) + router.On("FindProviders", mock.Anything, cid, 0).Return(routerResultIter, c.routerErr) } else { - router.On("FindProviders", mock.Anything, cid, 20).Return(findProvsIter, c.routerErr) + router.On("FindProviders", mock.Anything, cid, 20).Return(routerResultIter, c.routerErr) } - provsIter, err := client.FindProviders(ctx, cid) - + resultIter, err := client.FindProviders(ctx, cid) c.expErrContains.errContains(t, err) - provs := iter.ReadAll[iter.Result[types.ProviderResponse]](provsIter) - assert.Equal(t, c.expProvs, provs) + results := iter.ReadAll[iter.Result[types.Record]](resultIter) + assert.Equal(t, c.expResult, results) }) } } @@ -416,7 +442,8 @@ func TestClient_Provide(t *testing.T) { deps.server.Close() } if c.mangleSignature { - client.afterSignCallback = func(req *types.WriteBitswapProviderRecord) { + //lint:ignore SA1019 // ignore staticcheck + client.afterSignCallback = func(req *types.WriteBitswapRecord) { mh, err := multihash.Encode([]byte("boom"), multihash.SHA2_256) require.NoError(t, err) mb, err := multibase.Encode(multibase.Base64, mh) @@ -426,6 +453,7 @@ func TestClient_Provide(t *testing.T) { } } + //lint:ignore SA1019 // ignore staticcheck expectedProvReq := &server.BitswapWriteProvideRequest{ Keys: c.cids, Timestamp: clock.Now().Truncate(time.Millisecond), @@ -457,6 +485,134 @@ func TestClient_Provide(t *testing.T) { } } +func TestClient_FindPeers(t *testing.T) { + peerRecord := makePeerRecord() + peerRecords := []iter.Result[types.Record]{ + {Val: &peerRecord}, + } + pid := *peerRecord.ID + + cases := []struct { + name string + httpStatusCode int + stopServer bool + routerResult []iter.Result[types.Record] + routerErr error + clientRequiresStreaming bool + serverStreamingDisabled bool + + expErrContains osErrContains + expResult []iter.Result[types.Record] + expStreamingResponse bool + expJSONResponse bool + }{ + { + name: "happy case", + routerResult: peerRecords, + expResult: peerRecords, + expStreamingResponse: true, + }, + { + name: "server doesn't support streaming", + routerResult: peerRecords, + expResult: peerRecords, + serverStreamingDisabled: true, + expJSONResponse: true, + }, + { + name: "client requires streaming but server doesn't support it", + serverStreamingDisabled: true, + clientRequiresStreaming: true, + expErrContains: osErrContains{expContains: "HTTP error with StatusCode=400: no supported content types"}, + }, + { + name: "returns an error if there's a non-200 response", + httpStatusCode: 500, + expErrContains: osErrContains{expContains: "HTTP error with StatusCode=500"}, + }, + { + name: "returns an error if the HTTP client returns a non-HTTP error", + stopServer: true, + expErrContains: osErrContains{ + expContains: "connect: connection refused", + expContainsWin: "connectex: No connection could be made because the target machine actively refused it.", + }, + }, + { + name: "returns no providers if the HTTP server returns a 404 respones", + httpStatusCode: 404, + expResult: nil, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + var ( + clientOpts []Option + serverOpts []server.Option + onRespReceived []func(*http.Response) + onReqReceived []func(*http.Request) + ) + + if c.serverStreamingDisabled { + serverOpts = append(serverOpts, server.WithStreamingResultsDisabled()) + } + + if c.clientRequiresStreaming { + clientOpts = append(clientOpts, WithStreamResultsRequired()) + onReqReceived = append(onReqReceived, func(r *http.Request) { + assert.Equal(t, mediaTypeNDJSON, r.Header.Get("Accept")) + }) + } + + if c.expStreamingResponse { + onRespReceived = append(onRespReceived, func(r *http.Response) { + assert.Equal(t, mediaTypeNDJSON, r.Header.Get("Content-Type")) + }) + } + + if c.expJSONResponse { + onRespReceived = append(onRespReceived, func(r *http.Response) { + assert.Equal(t, mediaTypeJSON, r.Header.Get("Content-Type")) + }) + } + + deps := makeTestDeps(t, clientOpts, serverOpts) + + deps.recordingHTTPClient.f = append(deps.recordingHTTPClient.f, onRespReceived...) + deps.recordingHandler.f = append(deps.recordingHandler.f, onReqReceived...) + + client := deps.client + router := deps.router + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + if c.httpStatusCode != 0 { + deps.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(c.httpStatusCode) + }) + } + + if c.stopServer { + deps.server.Close() + } + + routerResultIter := iter.FromSlice(c.routerResult) + if c.expStreamingResponse { + router.On("FindPeers", mock.Anything, pid, 0).Return(routerResultIter, c.routerErr) + } else { + router.On("FindPeers", mock.Anything, pid, 20).Return(routerResultIter, c.routerErr) + } + + resultIter, err := client.FindPeers(ctx, pid) + c.expErrContains.errContains(t, err) + + results := iter.ReadAll[iter.Result[types.Record]](resultIter) + assert.Equal(t, c.expResult, results) + }) + } +} + func makeName(t *testing.T) (crypto.PrivKey, ipns.Name) { sk, _, err := crypto.GenerateEd25519Key(rand.Reader) require.NoError(t, err) @@ -492,9 +648,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("FindIPNSRecord", mock.Anything, name).Return(nil, errors.New("something wrong happened")) + router.On("GetIPNS", mock.Anything, name).Return(nil, errors.New("something wrong happened")) - receivedRecord, err := client.FindIPNSRecord(context.Background(), name) + receivedRecord, err := client.GetIPNS(context.Background(), name) require.Error(t, err) require.Nil(t, receivedRecord) }) @@ -508,9 +664,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("FindIPNSRecord", mock.Anything, name).Return(record, nil) + router.On("GetIPNS", mock.Anything, name).Return(record, nil) - receivedRecord, err := client.FindIPNSRecord(context.Background(), name) + receivedRecord, err := client.GetIPNS(context.Background(), name) require.NoError(t, err) require.Equal(t, record, receivedRecord) }) @@ -524,9 +680,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("FindIPNSRecord", mock.Anything, name2).Return(record, nil) + router.On("GetIPNS", mock.Anything, name2).Return(record, nil) - receivedRecord, err := client.FindIPNSRecord(context.Background(), name2) + receivedRecord, err := client.GetIPNS(context.Background(), name2) require.Error(t, err) require.Nil(t, receivedRecord) }) @@ -539,9 +695,9 @@ func TestClient_IPNS(t *testing.T) { client := deps.client router := deps.router - router.On("ProvideIPNSRecord", mock.Anything, name, record).Return(nil) + router.On("PutIPNS", mock.Anything, name, record).Return(nil) - err := client.ProvideIPNSRecord(context.Background(), name, record) + err := client.PutIPNS(context.Background(), name, record) require.NoError(t, err) }) } diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index 8318a3163..2438d4fea 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -3,26 +3,32 @@ package contentrouter import ( "context" "reflect" + "strings" "time" + "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/routing/http/internal" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" + routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multihash" ) -var logger = logging.Logger("service/contentrouting") +var logger = logging.Logger("routing/http/contentrouter") const ttl = 24 * time.Hour type Client interface { + FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) - FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.ProviderResponse], error) + FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) + GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) + PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error } type contentRouter struct { @@ -32,6 +38,10 @@ type contentRouter struct { } var _ routing.ContentRouting = (*contentRouter)(nil) +var _ routing.PeerRouting = (*contentRouter)(nil) +var _ routing.ValueStore = (*contentRouter)(nil) +var _ routinghelpers.ProvideManyRouter = (*contentRouter)(nil) +var _ routinghelpers.ReadyAbleRouter = (*contentRouter)(nil) type option func(c *contentRouter) @@ -60,8 +70,7 @@ func NewContentRoutingClient(c Client, opts ...option) *contentRouter { } func (c *contentRouter) Provide(ctx context.Context, key cid.Cid, announce bool) error { - // If 'true' is - // passed, it also announces it, otherwise it is just kept in the local + // If 'true' is passed, it also announces it, otherwise it is just kept in the local // accounting of which objects are being provided. if !announce { return nil @@ -73,7 +82,7 @@ func (c *contentRouter) Provide(ctx context.Context, key cid.Cid, announce bool) // ProvideMany provides a set of keys to the remote delegate. // Large sets of keys are chunked into multiple requests and sent concurrently, according to the concurrency configuration. -// TODO: implement retries through transient errors +// TODO: switch to use [client.Provide] when ready. func (c *contentRouter) ProvideMany(ctx context.Context, mhKeys []multihash.Multihash) error { keys := make([]cid.Cid, 0, len(mhKeys)) for _, m := range mhKeys { @@ -97,13 +106,14 @@ func (c *contentRouter) ProvideMany(ctx context.Context, mhKeys []multihash.Mult ) } -// Ready is part of the existing `ProvideMany` interface. +// Ready is part of the existing [routing.ReadyAbleRouter] interface. func (c *contentRouter) Ready() bool { return true } -// readProviderResponses reads bitswap records from the iterator into the given channel, dropping non-bitswap records. -func readProviderResponses(iter iter.ResultIter[types.ProviderResponse], ch chan<- peer.AddrInfo) { +// readProviderResponses reads peer records (and bitswap records for legacy +// compatibility) from the iterator into the given channel. +func readProviderResponses(iter iter.ResultIter[types.Record], ch chan<- peer.AddrInfo) { defer close(ch) defer iter.Close() for iter.Next() { @@ -113,8 +123,31 @@ func readProviderResponses(iter iter.ResultIter[types.ProviderResponse], ch chan continue } v := res.Val - if v.GetSchema() == types.SchemaBitswap { - result, ok := v.(*types.ReadBitswapProviderRecord) + switch v.GetSchema() { + case types.SchemaPeer: + result, ok := v.(*types.PeerRecord) + if !ok { + logger.Errorw( + "problem casting find providers result", + "Schema", v.GetSchema(), + "Type", reflect.TypeOf(v).String(), + ) + continue + } + + var addrs []multiaddr.Multiaddr + for _, a := range result.Addrs { + addrs = append(addrs, a.Multiaddr) + } + + ch <- peer.AddrInfo{ + ID: *result.ID, + Addrs: addrs, + } + //lint:ignore SA1019 // ignore staticcheck + case types.SchemaBitswap: + //lint:ignore SA1019 // ignore staticcheck + result, ok := v.(*types.BitswapRecord) if !ok { logger.Errorw( "problem casting find providers result", @@ -149,3 +182,111 @@ func (c *contentRouter) FindProvidersAsync(ctx context.Context, key cid.Cid, num go readProviderResponses(resultsIter, ch) return ch } + +func (c *contentRouter) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) { + iter, err := c.client.FindPeers(ctx, pid) + if err != nil { + return peer.AddrInfo{}, err + } + defer iter.Close() + + for iter.Next() { + res := iter.Val() + if res.Err != nil { + logger.Warnw("error iterating provider responses: %s", res.Err) + continue + } + v := res.Val + if v.GetSchema() == types.SchemaPeer { + result, ok := v.(*types.PeerRecord) + if !ok { + logger.Errorw( + "problem casting find providers result", + "Schema", v.GetSchema(), + "Type", reflect.TypeOf(v).String(), + ) + continue + } + + var addrs []multiaddr.Multiaddr + for _, a := range result.Addrs { + addrs = append(addrs, a.Multiaddr) + } + + return peer.AddrInfo{ + ID: *result.ID, + Addrs: addrs, + }, nil + } + } + + return peer.AddrInfo{}, err +} + +func (c *contentRouter) PutValue(ctx context.Context, key string, data []byte, opts ...routing.Option) error { + if !strings.HasPrefix(key, "/ipns/") { + return routing.ErrNotSupported + } + + name, err := ipns.NameFromRoutingKey([]byte(key)) + if err != nil { + return err + } + + record, err := ipns.UnmarshalRecord(data) + if err != nil { + return err + } + + return c.client.PutIPNS(ctx, name, record) +} + +func (c *contentRouter) GetValue(ctx context.Context, key string, opts ...routing.Option) ([]byte, error) { + if !strings.HasPrefix(key, "/ipns/") { + return nil, routing.ErrNotSupported + } + + name, err := ipns.NameFromRoutingKey([]byte(key)) + if err != nil { + return nil, err + } + + record, err := c.client.GetIPNS(ctx, name) + if err != nil { + return nil, err + } + + return ipns.MarshalRecord(record) +} + +func (c *contentRouter) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) { + if !strings.HasPrefix(key, "/ipns/") { + return nil, routing.ErrNotSupported + } + + name, err := ipns.NameFromRoutingKey([]byte(key)) + if err != nil { + return nil, err + } + + ch := make(chan []byte) + + go func() { + record, err := c.client.GetIPNS(ctx, name) + if err != nil { + close(ch) + return + } + + raw, err := ipns.MarshalRecord(record) + if err != nil { + close(ch) + return + } + + ch <- raw + close(ch) + }() + + return ch, nil +} diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index 3830482e2..83a086997 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -6,10 +6,15 @@ import ( "testing" "time" + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/ipns" + ipfspath "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -23,9 +28,14 @@ func (m *mockClient) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl tim return args.Get(0).(time.Duration), args.Error(1) } -func (m *mockClient) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.ProviderResponse], error) { +func (m *mockClient) FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key) - return args.Get(0).(iter.ResultIter[types.ProviderResponse]), args.Error(1) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) +} + +func (m *mockClient) FindPeers(ctx context.Context, pid peer.ID) (iter.ResultIter[types.Record], error) { + args := m.Called(ctx, pid) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } func (m *mockClient) Ready(ctx context.Context) (bool, error) { @@ -33,18 +43,14 @@ func (m *mockClient) Ready(ctx context.Context) (bool, error) { return args.Bool(0), args.Error(1) } -func makeCID() cid.Cid { - buf := make([]byte, 63) - _, err := rand.Read(buf) - if err != nil { - panic(err) - } - mh, err := multihash.Encode(buf, multihash.SHA2_256) - if err != nil { - panic(err) - } - c := cid.NewCidV1(cid.Raw, mh) - return c +func (m *mockClient) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { + args := m.Called(ctx, name) + return args.Get(0).(*ipns.Record), args.Error(1) +} + +func (m *mockClient) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { + args := m.Called(ctx, name, record) + return args.Error(0) } func TestProvide(t *testing.T) { @@ -100,6 +106,20 @@ func TestProvideMany(t *testing.T) { require.NoError(t, err) } +func makeCID() cid.Cid { + buf := make([]byte, 63) + _, err := rand.Read(buf) + if err != nil { + panic(err) + } + mh, err := multihash.Encode(buf, multihash.SHA2_256) + if err != nil { + panic(err) + } + c := cid.NewCidV1(cid.Raw, mh) + return c +} + func TestFindProvidersAsync(t *testing.T) { key := makeCID() ctx := context.Background() @@ -108,26 +128,40 @@ func TestFindProvidersAsync(t *testing.T) { p1 := peer.ID("peer1") p2 := peer.ID("peer2") - ais := []types.ProviderResponse{ - &types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", - Schema: types.SchemaBitswap, - ID: &p1, + p3 := peer.ID("peer3") + p4 := peer.ID("peer4") + ais := []types.Record{ + &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &p1, + Protocols: []string{"transport-bitswap"}, }, - &types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", + //lint:ignore SA1019 // ignore staticcheck + &types.BitswapRecord{ + //lint:ignore SA1019 // ignore staticcheck Schema: types.SchemaBitswap, ID: &p2, + Protocol: "transport-bitswap", + }, + &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &p3, + Protocols: []string{"transport-bitswap"}, + }, + &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &p4, + Protocols: []string{"transport-horse"}, }, - &types.UnknownProviderRecord{ - Protocol: "UNKNOWN", + &types.UnknownRecord{ + Schema: "unknown", }, } - aisIter := iter.ToResultIter[types.ProviderResponse](iter.FromSlice(ais)) + aisIter := iter.ToResultIter[types.Record](iter.FromSlice(ais)) client.On("FindProviders", ctx, key).Return(aisIter, nil) - aiChan := crc.FindProvidersAsync(ctx, key, 2) + aiChan := crc.FindProvidersAsync(ctx, key, 3) var actualAIs []peer.AddrInfo for ai := range aiChan { @@ -137,7 +171,118 @@ func TestFindProvidersAsync(t *testing.T) { expected := []peer.AddrInfo{ {ID: p1}, {ID: p2}, + {ID: p3}, + {ID: p4}, } require.Equal(t, expected, actualAIs) } + +func TestFindPeer(t *testing.T) { + ctx := context.Background() + client := &mockClient{} + crc := NewContentRoutingClient(client) + + p1 := peer.ID("peer1") + ais := []types.Record{ + &types.UnknownRecord{ + Schema: "unknown", + }, + &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &p1, + Protocols: []string{"transport-bitswap"}, + }, + } + aisIter := iter.ToResultIter[types.Record](iter.FromSlice(ais)) + + client.On("FindPeers", ctx, p1).Return(aisIter, nil) + + peer, err := crc.FindPeer(ctx, p1) + require.NoError(t, err) + require.Equal(t, peer.ID, p1) +} + +func makeName(t *testing.T) (crypto.PrivKey, ipns.Name) { + sk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + pid, err := peer.IDFromPrivateKey(sk) + require.NoError(t, err) + + return sk, ipns.NameFromPeer(pid) +} + +func makeIPNSRecord(t *testing.T, sk crypto.PrivKey, opts ...ipns.Option) (*ipns.Record, []byte) { + cid, err := cid.Decode("bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") + require.NoError(t, err) + + path := path.IpfsPath(cid) + eol := time.Now().Add(time.Hour * 48) + ttl := time.Second * 20 + + record, err := ipns.NewRecord(sk, ipfspath.FromString(path.String()), 1, eol, ttl, opts...) + require.NoError(t, err) + + rawRecord, err := ipns.MarshalRecord(record) + require.NoError(t, err) + + return record, rawRecord +} + +func TestGetValue(t *testing.T) { + ctx := context.Background() + client := &mockClient{} + crc := NewContentRoutingClient(client) + + t.Run("Fail On Unsupported Key", func(t *testing.T) { + v, err := crc.GetValue(ctx, "/something/unsupported") + require.Nil(t, v) + require.ErrorIs(t, err, routing.ErrNotSupported) + }) + + t.Run("Fail On Invalid IPNS Name", func(t *testing.T) { + v, err := crc.GetValue(ctx, "/ipns/invalid") + require.Nil(t, v) + require.Error(t, err) + }) + + t.Run("Succeeds On Valid IPNS Name", func(t *testing.T) { + sk, name := makeName(t) + rec, rawRec := makeIPNSRecord(t, sk) + client.On("GetIPNS", ctx, name).Return(rec, nil) + v, err := crc.GetValue(ctx, string(name.RoutingKey())) + require.NoError(t, err) + require.Equal(t, rawRec, v) + }) +} + +func TestPutValue(t *testing.T) { + ctx := context.Background() + client := &mockClient{} + crc := NewContentRoutingClient(client) + + sk, name := makeName(t) + _, rawRec := makeIPNSRecord(t, sk) + + t.Run("Fail On Unsupported Key", func(t *testing.T) { + err := crc.PutValue(ctx, "/something/unsupported", rawRec) + require.ErrorIs(t, err, routing.ErrNotSupported) + }) + + t.Run("Fail On Invalid IPNS Name", func(t *testing.T) { + err := crc.PutValue(ctx, "/ipns/invalid", rawRec) + require.Error(t, err) + }) + + t.Run("Fail On Invalid IPNS Record", func(t *testing.T) { + err := crc.PutValue(ctx, string(name.RoutingKey()), []byte("gibberish")) + require.Error(t, err) + }) + + t.Run("Succeeds On Valid IPNS Name & Record", func(t *testing.T) { + client.On("PutIPNS", ctx, name, mock.Anything).Return(nil) + err := crc.PutValue(ctx, string(name.RoutingKey()), rawRec) + require.NoError(t, err) + }) +} diff --git a/routing/http/server/server.go b/routing/http/server/server.go index 835262990..9e7d81a04 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -37,34 +37,45 @@ const ( DefaultStreamingRecordsLimit = 0 ) -var logger = logging.Logger("service/server/delegatedrouting") +var logger = logging.Logger("routing/http/server") const ( - ProvidePath = "/routing/v1/providers/" - FindProvidersPath = "/routing/v1/providers/{cid}" - IPNSPath = "/routing/v1/ipns/{cid}" + providePath = "/routing/v1/providers/" + findProvidersPath = "/routing/v1/providers/{cid}" + findPeersPath = "/routing/v1/peers/{peer-id}" + GetIPNSPath = "/routing/v1/ipns/{cid}" ) type FindProvidersAsyncResponse struct { - ProviderResponse types.ProviderResponse + ProviderResponse types.Record Error error } type ContentRouter interface { - // FindProviders searches for peers who are able to provide a given key. Limit - // indicates the maximum amount of results to return. 0 means unbounded. - FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) + // FindProviders searches for peers who are able to provide the given [cid.Cid]. + // Limit indicates the maximum amount of results to return; 0 means unbounded. + FindProviders(ctx context.Context, cid cid.Cid, limit int) (iter.ResultIter[types.Record], error) + + // Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: + // + // [IPIP-378]: https://github.com/ipfs/specs/pull/378 ProvideBitswap(ctx context.Context, req *BitswapWriteProvideRequest) (time.Duration, error) - Provide(ctx context.Context, req *WriteProvideRequest) (types.ProviderResponse, error) - // FindIPNSRecord searches for an [ipns.Record] for the given [ipns.Name]. - FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) + // FindPeers searches for peers who have the provided [peer.ID]. + // Limit indicates the maximum amount of results to return; 0 means unbounded. + FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) + + // GetIPNS searches for an [ipns.Record] for the given [ipns.Name]. + GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) - // ProvideIPNSRecord stores the provided [ipns.Record] for the given [ipns.Name]. It is - // guaranteed that the record matches the provided name. - ProvideIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error + // PutIPNS stores the provided [ipns.Record] for the given [ipns.Name]. + // It is guaranteed that the record matches the provided name. + PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error } +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 type BitswapWriteProvideRequest struct { Keys []cid.Cid Timestamp time.Time @@ -73,6 +84,9 @@ type BitswapWriteProvideRequest struct { Addrs []multiaddr.Multiaddr } +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 type WriteProvideRequest struct { Protocol string Schema string @@ -88,16 +102,18 @@ func WithStreamingResultsDisabled() Option { } } -// WithRecordsLimit sets a limit that will be passed to ContentRouter.FindProviders -// for non-streaming requests (application/json). Default is DefaultRecordsLimit. +// WithRecordsLimit sets a limit that will be passed to [ContentRouter.FindProviders] +// and [ContentRouter.FindPeers] for non-streaming requests (application/json). +// Default is [DefaultRecordsLimit]. func WithRecordsLimit(limit int) Option { return func(s *server) { s.recordsLimit = limit } } -// WithStreamingRecordsLimit sets a limit that will be passed to ContentRouter.FindProviders -// for streaming requests (application/x-ndjson). Default is DefaultStreamingRecordsLimit. +// WithStreamingRecordsLimit sets a limit that will be passed to [ContentRouter.FindProviders] +// and [ContentRouter.FindPeers] for streaming requests (application/x-ndjson). +// Default is [DefaultStreamingRecordsLimit]. func WithStreamingRecordsLimit(limit int) Option { return func(s *server) { s.streamingRecordsLimit = limit @@ -116,12 +132,11 @@ func Handler(svc ContentRouter, opts ...Option) http.Handler { } r := mux.NewRouter() - r.HandleFunc(ProvidePath, server.provide).Methods(http.MethodPut) - r.HandleFunc(FindProvidersPath, server.findProviders).Methods(http.MethodGet) - - r.HandleFunc(IPNSPath, server.getIPNSRecord).Methods(http.MethodGet) - r.HandleFunc(IPNSPath, server.putIPNSRecord).Methods(http.MethodPut) - + r.HandleFunc(findProvidersPath, server.findProviders).Methods(http.MethodGet) + r.HandleFunc(providePath, server.provide).Methods(http.MethodPut) + r.HandleFunc(findPeersPath, server.findPeers).Methods(http.MethodGet) + r.HandleFunc(GetIPNSPath, server.GetIPNS).Methods(http.MethodGet) + r.HandleFunc(GetIPNSPath, server.PutIPNS).Methods(http.MethodPut) return r } @@ -132,7 +147,149 @@ type server struct { streamingRecordsLimit int } +func (s *server) detectResponseType(r *http.Request) (string, error) { + var ( + supportsNDJSON bool + supportsJSON bool + + acceptHeaders = r.Header.Values("Accept") + ) + + if len(acceptHeaders) == 0 { + return mediaTypeJSON, nil + } + + for _, acceptHeader := range acceptHeaders { + for _, accept := range strings.Split(acceptHeader, ",") { + mediaType, _, err := mime.ParseMediaType(accept) + if err != nil { + return "", fmt.Errorf("unable to parse Accept header: %w", err) + } + + switch mediaType { + case mediaTypeJSON, mediaTypeWildcard: + supportsJSON = true + case mediaTypeNDJSON: + supportsNDJSON = true + } + } + } + + if supportsNDJSON && !s.disableNDJSON { + return mediaTypeNDJSON, nil + } else if supportsJSON { + return mediaTypeJSON, nil + } else { + return "", errors.New("no supported content types") + } +} + +func (s *server) findProviders(w http.ResponseWriter, httpReq *http.Request) { + vars := mux.Vars(httpReq) + cidStr := vars["cid"] + cid, err := cid.Decode(cidStr) + if err != nil { + writeErr(w, "FindProviders", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + return + } + + mediaType, err := s.detectResponseType(httpReq) + if err != nil { + writeErr(w, "FindProviders", http.StatusBadRequest, err) + return + } + + var ( + handlerFunc func(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) + recordsLimit int + ) + + if mediaType == mediaTypeNDJSON { + handlerFunc = s.findProvidersNDJSON + recordsLimit = s.streamingRecordsLimit + } else { + handlerFunc = s.findProvidersJSON + recordsLimit = s.recordsLimit + } + + provIter, err := s.svc.FindProviders(httpReq.Context(), cid, recordsLimit) + if err != nil { + writeErr(w, "FindProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + return + } + + handlerFunc(w, provIter) +} + +func (s *server) findProvidersJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { + defer provIter.Close() + + providers, err := iter.ReadAllResults(provIter) + if err != nil { + writeErr(w, "FindProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + return + } + + writeJSONResult(w, "FindProviders", jsontypes.ProvidersResponse{ + Providers: providers, + }) +} + +func (s *server) findProvidersNDJSON(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) { + writeResultsIterNDJSON(w, provIter) +} + +func (s *server) findPeers(w http.ResponseWriter, r *http.Request) { + pidStr := mux.Vars(r)["peer-id"] + + // pidStr must be in CIDv1 format. Therefore, use [cid.Decode]. We can't use + // [peer.Decode] because that would allow other formats to pass through. + cid, err := cid.Decode(pidStr) + if err != nil { + if pid, err := peer.Decode(pidStr); err == nil { + writeErr(w, "FindPeers", http.StatusBadRequest, fmt.Errorf("the value is a peer ID, try using its CID representation: %s", peer.ToCid(pid).String())) + } else { + writeErr(w, "FindPeers", http.StatusBadRequest, fmt.Errorf("unable to parse peer ID: %w", err)) + } + return + } + + pid, err := peer.FromCid(cid) + if err != nil { + writeErr(w, "FindPeers", http.StatusBadRequest, fmt.Errorf("unable to parse peer ID: %w", err)) + return + } + + mediaType, err := s.detectResponseType(r) + if err != nil { + writeErr(w, "FindPeers", http.StatusBadRequest, err) + return + } + + var ( + handlerFunc func(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) + recordsLimit int + ) + + if mediaType == mediaTypeNDJSON { + handlerFunc = s.findPeersNDJSON + recordsLimit = s.streamingRecordsLimit + } else { + handlerFunc = s.findPeersJSON + recordsLimit = s.recordsLimit + } + + provIter, err := s.svc.FindPeers(r.Context(), pid, recordsLimit) + if err != nil { + writeErr(w, "FindPeers", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + return + } + + handlerFunc(w, provIter) +} + func (s *server) provide(w http.ResponseWriter, httpReq *http.Request) { + //lint:ignore SA1019 // ignore staticcheck req := jsontypes.WriteProvidersRequest{} err := json.NewDecoder(httpReq.Body).Decode(&req) _ = httpReq.Body.Close() @@ -141,11 +298,13 @@ func (s *server) provide(w http.ResponseWriter, httpReq *http.Request) { return } + //lint:ignore SA1019 // ignore staticcheck resp := jsontypes.WriteProvidersResponse{} for i, prov := range req.Providers { switch v := prov.(type) { - case *types.WriteBitswapProviderRecord: + //lint:ignore SA1019 // ignore staticcheck + case *types.WriteBitswapRecord: err := v.Verify() if err != nil { logErr("Provide", "signature verification failed", err) @@ -173,148 +332,42 @@ func (s *server) provide(w http.ResponseWriter, httpReq *http.Request) { return } resp.ProvideResults = append(resp.ProvideResults, - &types.WriteBitswapProviderRecordResponse{ + //lint:ignore SA1019 // ignore staticcheck + &types.WriteBitswapRecordResponse{ Protocol: v.Protocol, Schema: v.Schema, AdvisoryTTL: &types.Duration{Duration: advisoryTTL}, }, ) - case *types.UnknownProviderRecord: - provResp, err := s.svc.Provide(httpReq.Context(), &WriteProvideRequest{ - Protocol: v.Protocol, - Schema: v.Schema, - Bytes: v.Bytes, - }) - if err != nil { - writeErr(w, "Provide", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) - return - } - resp.ProvideResults = append(resp.ProvideResults, provResp) default: - writeErr(w, "Provide", http.StatusBadRequest, fmt.Errorf("provider record %d does not contain a protocol", i)) + writeErr(w, "Provide", http.StatusBadRequest, fmt.Errorf("provider record %d is not bitswap", i)) return } } writeJSONResult(w, "Provide", resp) } -func (s *server) findProviders(w http.ResponseWriter, httpReq *http.Request) { - vars := mux.Vars(httpReq) - cidStr := vars["cid"] - cid, err := cid.Decode(cidStr) - if err != nil { - writeErr(w, "FindProviders", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) - return - } - - var handlerFunc func(w http.ResponseWriter, provIter iter.ResultIter[types.ProviderResponse]) - - var supportsNDJSON bool - var supportsJSON bool - var recordsLimit int - acceptHeaders := httpReq.Header.Values("Accept") - if len(acceptHeaders) == 0 { - handlerFunc = s.findProvidersJSON - recordsLimit = s.recordsLimit - } else { - for _, acceptHeader := range acceptHeaders { - for _, accept := range strings.Split(acceptHeader, ",") { - mediaType, _, err := mime.ParseMediaType(accept) - if err != nil { - writeErr(w, "FindProviders", http.StatusBadRequest, fmt.Errorf("unable to parse Accept header: %w", err)) - return - } - - switch mediaType { - case mediaTypeJSON, mediaTypeWildcard: - supportsJSON = true - case mediaTypeNDJSON: - supportsNDJSON = true - } - } - } - - if supportsNDJSON && !s.disableNDJSON { - handlerFunc = s.findProvidersNDJSON - recordsLimit = s.streamingRecordsLimit - } else if supportsJSON { - handlerFunc = s.findProvidersJSON - recordsLimit = s.recordsLimit - } else { - writeErr(w, "FindProviders", http.StatusBadRequest, errors.New("no supported content types")) - return - } - } +func (s *server) findPeersJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { + defer peersIter.Close() - provIter, err := s.svc.FindProviders(httpReq.Context(), cid, recordsLimit) + peers, err := iter.ReadAllResults(peersIter) if err != nil { - writeErr(w, "FindProviders", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "FindPeers", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } - handlerFunc(w, provIter) + writeJSONResult(w, "FindPeers", jsontypes.PeersResponse{ + Peers: peers, + }) } -func (s *server) findProvidersJSON(w http.ResponseWriter, provIter iter.ResultIter[types.ProviderResponse]) { - defer provIter.Close() - - var ( - providers []types.ProviderResponse - i int - ) - - for provIter.Next() { - res := provIter.Val() - if res.Err != nil { - writeErr(w, "FindProviders", http.StatusInternalServerError, fmt.Errorf("delegate error on result %d: %w", i, res.Err)) - return - } - providers = append(providers, res.Val) - i++ - } - response := jsontypes.ReadProvidersResponse{Providers: providers} - writeJSONResult(w, "FindProviders", response) -} - -func (s *server) findProvidersNDJSON(w http.ResponseWriter, provIter iter.ResultIter[types.ProviderResponse]) { - defer provIter.Close() - - w.Header().Set("Content-Type", mediaTypeNDJSON) - w.WriteHeader(http.StatusOK) - for provIter.Next() { - res := provIter.Val() - if res.Err != nil { - logger.Errorw("FindProviders ndjson iterator error", "Error", res.Err) - return - } - // don't use an encoder because we can't easily differentiate writer errors from encoding errors - b, err := drjson.MarshalJSONBytes(res.Val) - if err != nil { - logger.Errorw("FindProviders ndjson marshal error", "Error", err) - return - } - - _, err = w.Write(b) - if err != nil { - logger.Warn("FindProviders ndjson write error", "Error", err) - return - } - - _, err = w.Write([]byte{'\n'}) - if err != nil { - logger.Warn("FindProviders ndjson write error", "Error", err) - return - } - - if f, ok := w.(http.Flusher); ok { - f.Flush() - } - } +func (s *server) findPeersNDJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { + writeResultsIterNDJSON(w, peersIter) } -func (s *server) getIPNSRecord(w http.ResponseWriter, r *http.Request) { +func (s *server) GetIPNS(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.Header.Get("Accept"), mediaTypeIPNSRecord) { - writeErr(w, "GetIPNSRecord", http.StatusNotAcceptable, errors.New("content type in 'Accept' header is missing or not supported")) + writeErr(w, "GetIPNS", http.StatusNotAcceptable, errors.New("content type in 'Accept' header is missing or not supported")) return } @@ -322,25 +375,25 @@ func (s *server) getIPNSRecord(w http.ResponseWriter, r *http.Request) { cidStr := vars["cid"] cid, err := cid.Decode(cidStr) if err != nil { - writeErr(w, "GetIPNSRecord", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + writeErr(w, "GetIPNS", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) return } name, err := ipns.NameFromCid(cid) if err != nil { - writeErr(w, "GetIPNSRecord", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) + writeErr(w, "GetIPNS", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) return } - record, err := s.svc.FindIPNSRecord(r.Context(), name) + record, err := s.svc.GetIPNS(r.Context(), name) if err != nil { - writeErr(w, "GetIPNSRecord", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "GetIPNS", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } rawRecord, err := ipns.MarshalRecord(record) if err != nil { - writeErr(w, "GetIPNSRecord", http.StatusInternalServerError, err) + writeErr(w, "GetIPNS", http.StatusInternalServerError, err) return } @@ -356,9 +409,9 @@ func (s *server) getIPNSRecord(w http.ResponseWriter, r *http.Request) { w.Write(rawRecord) } -func (s *server) putIPNSRecord(w http.ResponseWriter, r *http.Request) { +func (s *server) PutIPNS(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.Header.Get("Content-Type"), mediaTypeIPNSRecord) { - writeErr(w, "PutIPNSRecord", http.StatusNotAcceptable, errors.New("content type in 'Content-Type' header is missing or not supported")) + writeErr(w, "PutIPNS", http.StatusNotAcceptable, errors.New("content type in 'Content-Type' header is missing or not supported")) return } @@ -366,38 +419,38 @@ func (s *server) putIPNSRecord(w http.ResponseWriter, r *http.Request) { cidStr := vars["cid"] cid, err := cid.Decode(cidStr) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) + writeErr(w, "PutIPNS", http.StatusBadRequest, fmt.Errorf("unable to parse CID: %w", err)) return } name, err := ipns.NameFromCid(cid) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) + writeErr(w, "PutIPNS", http.StatusBadRequest, fmt.Errorf("peer ID CID is not valid: %w", err)) return } // Limit the reader to the maximum record size. rawRecord, err := io.ReadAll(io.LimitReader(r.Body, int64(ipns.MaxRecordSize))) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("provided record is too long: %w", err)) + writeErr(w, "PutIPNS", http.StatusBadRequest, fmt.Errorf("provided record is too long: %w", err)) return } record, err := ipns.UnmarshalRecord(rawRecord) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) + writeErr(w, "PutIPNS", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) return } err = ipns.ValidateWithName(record, name) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) + writeErr(w, "PutIPNS", http.StatusBadRequest, fmt.Errorf("provided record is invalid: %w", err)) return } - err = s.svc.ProvideIPNSRecord(r.Context(), name, record) + err = s.svc.PutIPNS(r.Context(), name, record) if err != nil { - writeErr(w, "PutIPNSRecord", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) + writeErr(w, "PutIPNS", http.StatusInternalServerError, fmt.Errorf("delegate error: %w", err)) return } @@ -437,3 +490,40 @@ func writeErr(w http.ResponseWriter, method string, statusCode int, cause error) func logErr(method, msg string, err error) { logger.Infow(msg, "Method", method, "Error", err) } + +func writeResultsIterNDJSON(w http.ResponseWriter, resultIter iter.ResultIter[types.Record]) { + defer resultIter.Close() + + w.Header().Set("Content-Type", mediaTypeNDJSON) + w.WriteHeader(http.StatusOK) + + for resultIter.Next() { + res := resultIter.Val() + if res.Err != nil { + logger.Errorw("ndjson iterator error", "Error", res.Err) + return + } + // don't use an encoder because we can't easily differentiate writer errors from encoding errors + b, err := drjson.MarshalJSONBytes(res.Val) + if err != nil { + logger.Errorw("ndjson marshal error", "Error", err) + return + } + + _, err = w.Write(b) + if err != nil { + logger.Warn("ndjson write error", "Error", err) + return + } + + _, err = w.Write([]byte{'\n'}) + if err != nil { + logger.Warn("ndjson write error", "Error", err) + return + } + + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + } +} diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index dfe38f0da..f6d4a3dba 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -18,6 +18,7 @@ import ( "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" + b58 "github.com/mr-tron/base58/base58" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -28,12 +29,11 @@ func TestHeaders(t *testing.T) { t.Cleanup(server.Close) serverAddr := "http://" + server.Listener.Addr().String() - results := iter.FromSlice([]iter.Result[types.ProviderResponse]{ - {Val: &types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", - Schema: types.SchemaBitswap, - }}, - }, + results := iter.FromSlice([]iter.Result[types.Record]{ + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + Protocols: []string{"transport-bitswap"}, + }}}, ) c := "baeabep4vu3ceru7nerjjbk37sxb7wmftteve4hcosmyolsbsiubw2vr6pqzj6mw7kv6tbn6nqkkldnklbjgm5tzbi4hkpkled4xlcr7xz4bq" @@ -43,13 +43,13 @@ func TestHeaders(t *testing.T) { router.On("FindProviders", mock.Anything, cb, DefaultRecordsLimit). Return(results, nil) - resp, err := http.Get(serverAddr + ProvidePath + c) + resp, err := http.Get(serverAddr + "/routing/v1/providers/" + c) require.NoError(t, err) require.Equal(t, 200, resp.StatusCode) header := resp.Header.Get("Content-Type") require.Equal(t, mediaTypeJSON, header) - resp, err = http.Get(serverAddr + ProvidePath + "BAD_CID") + resp, err = http.Get(serverAddr + "/routing/v1/providers/" + "BAD_CID") require.NoError(t, err) defer resp.Body.Close() require.Equal(t, 400, resp.StatusCode) @@ -57,7 +57,17 @@ func TestHeaders(t *testing.T) { require.Equal(t, "text/plain; charset=utf-8", header) } -func TestResponse(t *testing.T) { +func makePeerID(t *testing.T) (crypto.PrivKey, peer.ID) { + sk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + pid, err := peer.IDFromPrivateKey(sk) + require.NoError(t, err) + + return sk, pid +} + +func TestProviders(t *testing.T) { pidStr := "12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn" pid2Str := "12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz" cidStr := "bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4" @@ -73,20 +83,21 @@ func TestResponse(t *testing.T) { runTest := func(t *testing.T, contentType string, expectedStream bool, expectedBody string) { t.Parallel() - results := iter.FromSlice([]iter.Result[types.ProviderResponse]{ - {Val: &types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", - Schema: types.SchemaBitswap, - ID: &pid, - Addrs: []types.Multiaddr{}, + results := iter.FromSlice([]iter.Result[types.Record]{ + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid, + Protocols: []string{"transport-bitswap"}, + Addrs: []types.Multiaddr{}, }}, - {Val: &types.ReadBitswapProviderRecord{ - Protocol: "transport-bitswap", + //lint:ignore SA1019 // ignore staticcheck + {Val: &types.BitswapRecord{ + //lint:ignore SA1019 // ignore staticcheck Schema: types.SchemaBitswap, ID: &pid2, + Protocol: "transport-bitswap", Addrs: []types.Multiaddr{}, - }}, - }, + }}}, ) router := &mockContentRouter{} @@ -98,7 +109,7 @@ func TestResponse(t *testing.T) { limit = DefaultStreamingRecordsLimit } router.On("FindProviders", mock.Anything, cid, limit).Return(results, nil) - urlStr := serverAddr + ProvidePath + cidStr + urlStr := serverAddr + "/routing/v1/providers/" + cidStr req, err := http.NewRequest(http.MethodGet, urlStr, nil) require.NoError(t, err) @@ -113,25 +124,120 @@ func TestResponse(t *testing.T) { body, err := io.ReadAll(resp.Body) require.NoError(t, err) - require.Equal(t, string(body), expectedBody) + require.Equal(t, expectedBody, string(body)) } t.Run("JSON Response", func(t *testing.T) { - runTest(t, mediaTypeJSON, false, `{"Providers":[{"Protocol":"transport-bitswap","Schema":"bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Addrs":[]},{"Protocol":"transport-bitswap","Schema":"bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Addrs":[]}]}`) + runTest(t, mediaTypeJSON, false, `{"Providers":[{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"},{"Schema":"bitswap","Protocol":"transport-bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz"}]}`) }) t.Run("NDJSON Response", func(t *testing.T) { - runTest(t, mediaTypeNDJSON, true, `{"Protocol":"transport-bitswap","Schema":"bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Addrs":[]}`+"\n"+`{"Protocol":"transport-bitswap","Schema":"bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz","Addrs":[]}`+"\n") + runTest(t, mediaTypeNDJSON, true, `{"Addrs":[],"ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Protocols":["transport-bitswap"],"Schema":"peer"}`+"\n"+`{"Schema":"bitswap","Protocol":"transport-bitswap","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vz"}`+"\n") }) } -func makeName(t *testing.T) (crypto.PrivKey, ipns.Name) { - sk, _, err := crypto.GenerateEd25519Key(rand.Reader) - require.NoError(t, err) +func TestPeers(t *testing.T) { + makeRequest := func(t *testing.T, router *mockContentRouter, contentType, arg string) *http.Response { + server := httptest.NewServer(Handler(router)) + t.Cleanup(server.Close) + req, err := http.NewRequest(http.MethodGet, "http://"+server.Listener.Addr().String()+"/routing/v1/peers/"+arg, nil) + require.NoError(t, err) + req.Header.Set("Accept", contentType) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + return resp + } - pid, err := peer.IDFromPrivateKey(sk) - require.NoError(t, err) + t.Run("GET /routing/v1/peers/{non-peer-cid} returns 400", func(t *testing.T) { + t.Parallel() + router := &mockContentRouter{} + resp := makeRequest(t, router, mediaTypeJSON, "bafkqaaa") + require.Equal(t, 400, resp.StatusCode) + }) + + t.Run("GET /routing/v1/peers/{base58-peer-id} returns 400", func(t *testing.T) { + t.Parallel() + + _, pid := makePeerID(t) + router := &mockContentRouter{} + resp := makeRequest(t, router, mediaTypeJSON, b58.Encode([]byte(pid))) + require.Equal(t, 400, resp.StatusCode) + }) + + t.Run("GET /routing/v1/peers/{cid-peer-id} returns 200 with correct body (JSON)", func(t *testing.T) { + t.Parallel() + + _, pid := makePeerID(t) + results := iter.FromSlice([]iter.Result[types.Record]{ + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid, + Protocols: []string{"transport-bitswap", "transport-foo"}, + Addrs: []types.Multiaddr{}, + }}, + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid, + Protocols: []string{"transport-foo"}, + Addrs: []types.Multiaddr{}, + }}, + }) + + router := &mockContentRouter{} + router.On("FindPeers", mock.Anything, pid, 20).Return(results, nil) + + resp := makeRequest(t, router, mediaTypeJSON, peer.ToCid(pid).String()) + require.Equal(t, 200, resp.StatusCode) + + header := resp.Header.Get("Content-Type") + require.Equal(t, mediaTypeJSON, header) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + expectedBody := `{"Peers":[{"Addrs":[],"ID":"` + pid.String() + `","Protocols":["transport-bitswap","transport-foo"],"Schema":"peer"},{"Addrs":[],"ID":"` + pid.String() + `","Protocols":["transport-foo"],"Schema":"peer"}]}` + require.Equal(t, expectedBody, string(body)) + }) + + t.Run("GET /routing/v1/peers/{cid-peer-id} returns 200 with correct body (NDJSON)", func(t *testing.T) { + t.Parallel() + + _, pid := makePeerID(t) + results := iter.FromSlice([]iter.Result[types.Record]{ + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid, + Protocols: []string{"transport-bitswap", "transport-foo"}, + Addrs: []types.Multiaddr{}, + }}, + {Val: &types.PeerRecord{ + Schema: types.SchemaPeer, + ID: &pid, + Protocols: []string{"transport-foo"}, + Addrs: []types.Multiaddr{}, + }}, + }) + + router := &mockContentRouter{} + router.On("FindPeers", mock.Anything, pid, 0).Return(results, nil) + + resp := makeRequest(t, router, mediaTypeNDJSON, peer.ToCid(pid).String()) + require.Equal(t, 200, resp.StatusCode) + + header := resp.Header.Get("Content-Type") + require.Equal(t, mediaTypeNDJSON, header) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + expectedBody := `{"Addrs":[],"ID":"` + pid.String() + `","Protocols":["transport-bitswap","transport-foo"],"Schema":"peer"}` + "\n" + `{"Addrs":[],"ID":"` + pid.String() + `","Protocols":["transport-foo"],"Schema":"peer"}` + "\n" + require.Equal(t, expectedBody, string(body)) + }) +} + +func makeName(t *testing.T) (crypto.PrivKey, ipns.Name) { + sk, pid := makePeerID(t) return sk, ipns.NameFromPeer(pid) } @@ -179,7 +285,7 @@ func TestIPNS(t *testing.T) { require.NoError(t, err) router := &mockContentRouter{} - router.On("FindIPNSRecord", mock.Anything, name1).Return(rec, nil) + router.On("GetIPNS", mock.Anything, name1).Return(rec, nil) resp := makeRequest(t, router, "/routing/v1/ipns/"+name1.String()) require.Equal(t, 200, resp.StatusCode) @@ -212,7 +318,7 @@ func TestIPNS(t *testing.T) { t.Parallel() router := &mockContentRouter{} - router.On("ProvideIPNSRecord", mock.Anything, name1, record1).Return(nil) + router.On("PutIPNS", mock.Anything, name1, record1).Return(nil) server := httptest.NewServer(Handler(router)) t.Cleanup(server.Close) @@ -259,9 +365,9 @@ func TestIPNS(t *testing.T) { type mockContentRouter struct{ mock.Mock } -func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.ProviderResponse], error) { +func (m *mockContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) { args := m.Called(ctx, key, limit) - return args.Get(0).(iter.ResultIter[types.ProviderResponse]), args.Error(1) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *BitswapWriteProvideRequest) (time.Duration, error) { @@ -269,17 +375,17 @@ func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *BitswapWrit return args.Get(0).(time.Duration), args.Error(1) } -func (m *mockContentRouter) Provide(ctx context.Context, req *WriteProvideRequest) (types.ProviderResponse, error) { - args := m.Called(ctx, req) - return args.Get(0).(types.ProviderResponse), args.Error(1) +func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { + args := m.Called(ctx, pid, limit) + return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockContentRouter) FindIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (m *mockContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { args := m.Called(ctx, name) return args.Get(0).(*ipns.Record), args.Error(1) } -func (m *mockContentRouter) ProvideIPNSRecord(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (m *mockContentRouter) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { args := m.Called(ctx, name, record) return args.Error(0) } diff --git a/routing/http/types/iter/iter.go b/routing/http/types/iter/iter.go index 67c6dde00..2e9801d46 100644 --- a/routing/http/types/iter/iter.go +++ b/routing/http/types/iter/iter.go @@ -1,5 +1,7 @@ package iter +import "fmt" + // Iter is an iterator of arbitrary values. // Iterators are generally not goroutine-safe, to make them safe just read from them into a channel. // For our use cases, these usually have a single reader. This motivates iterators instead of channels, @@ -44,3 +46,21 @@ func ReadAll[T any](iter Iter[T]) []T { } return vs } + +func ReadAllResults[T any](iter ResultIter[T]) ([]T, error) { + var ( + vs []T + i int + ) + + for iter.Next() { + res := iter.Val() + if res.Err != nil { + return nil, fmt.Errorf("error on result %d: %w", i, res.Err) + } + vs = append(vs, res.Val) + i++ + } + + return vs, nil +} diff --git a/routing/http/types/json/provider.go b/routing/http/types/json/provider.go deleted file mode 100644 index 351197338..000000000 --- a/routing/http/types/json/provider.go +++ /dev/null @@ -1,116 +0,0 @@ -package json - -import ( - "encoding/json" - - "github.com/ipfs/boxo/routing/http/types" -) - -// ReadProvidersResponse is the result of a Provide request -type ReadProvidersResponse struct { - Providers []types.ProviderResponse -} - -func (r *ReadProvidersResponse) UnmarshalJSON(b []byte) error { - var tempFPR struct{ Providers []json.RawMessage } - err := json.Unmarshal(b, &tempFPR) - if err != nil { - return err - } - - for _, provBytes := range tempFPR.Providers { - var readProv types.UnknownProviderRecord - err := json.Unmarshal(provBytes, &readProv) - if err != nil { - return err - } - - switch readProv.Schema { - case types.SchemaBitswap: - var prov types.ReadBitswapProviderRecord - err := json.Unmarshal(readProv.Bytes, &prov) - if err != nil { - return err - } - r.Providers = append(r.Providers, &prov) - default: - r.Providers = append(r.Providers, &readProv) - } - - } - return nil -} - -type WriteProvidersRequest struct { - Providers []types.WriteProviderRecord -} - -func (r *WriteProvidersRequest) UnmarshalJSON(b []byte) error { - type wpr struct{ Providers []json.RawMessage } - var tempWPR wpr - err := json.Unmarshal(b, &tempWPR) - if err != nil { - return err - } - - for _, provBytes := range tempWPR.Providers { - var rawProv types.UnknownProviderRecord - err := json.Unmarshal(provBytes, &rawProv) - if err != nil { - return err - } - - switch rawProv.Schema { - case types.SchemaBitswap: - var prov types.WriteBitswapProviderRecord - err := json.Unmarshal(rawProv.Bytes, &prov) - if err != nil { - return err - } - r.Providers = append(r.Providers, &prov) - default: - var prov types.UnknownProviderRecord - err := json.Unmarshal(b, &prov) - if err != nil { - return err - } - r.Providers = append(r.Providers, &prov) - } - } - return nil -} - -// WriteProvidersResponse is the result of a Provide operation -type WriteProvidersResponse struct { - ProvideResults []types.ProviderResponse -} - -func (r *WriteProvidersResponse) UnmarshalJSON(b []byte) error { - var tempWPR struct{ ProvideResults []json.RawMessage } - err := json.Unmarshal(b, &tempWPR) - if err != nil { - return err - } - - for _, provBytes := range tempWPR.ProvideResults { - var rawProv types.UnknownProviderRecord - err := json.Unmarshal(provBytes, &rawProv) - if err != nil { - return err - } - - switch rawProv.Schema { - case types.SchemaBitswap: - var prov types.WriteBitswapProviderRecordResponse - err := json.Unmarshal(rawProv.Bytes, &prov) - if err != nil { - return err - } - r.ProvideResults = append(r.ProvideResults, &prov) - default: - r.ProvideResults = append(r.ProvideResults, &rawProv) - } - } - - return nil -} diff --git a/routing/http/types/json/requests.go b/routing/http/types/json/requests.go new file mode 100644 index 000000000..4b582c3ba --- /dev/null +++ b/routing/http/types/json/requests.go @@ -0,0 +1,51 @@ +package json + +import ( + "encoding/json" + + "github.com/ipfs/boxo/routing/http/types" +) + +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +type WriteProvidersRequest struct { + Providers []types.Record +} + +func (r *WriteProvidersRequest) UnmarshalJSON(b []byte) error { + type wpr struct{ Providers []json.RawMessage } + var tempWPR wpr + err := json.Unmarshal(b, &tempWPR) + if err != nil { + return err + } + + for _, provBytes := range tempWPR.Providers { + var rawProv types.UnknownRecord + err := json.Unmarshal(provBytes, &rawProv) + if err != nil { + return err + } + + switch rawProv.Schema { + //lint:ignore SA1019 // ignore staticcheck + case types.SchemaBitswap: + //lint:ignore SA1019 // ignore staticcheck + var prov types.WriteBitswapRecord + err := json.Unmarshal(rawProv.Bytes, &prov) + if err != nil { + return err + } + r.Providers = append(r.Providers, &prov) + default: + var prov types.UnknownRecord + err := json.Unmarshal(b, &prov) + if err != nil { + return err + } + r.Providers = append(r.Providers, &prov) + } + } + return nil +} diff --git a/routing/http/types/json/responses.go b/routing/http/types/json/responses.go new file mode 100644 index 000000000..dfcfad830 --- /dev/null +++ b/routing/http/types/json/responses.go @@ -0,0 +1,98 @@ +package json + +import ( + "encoding/json" + + "github.com/ipfs/boxo/routing/http/types" +) + +// ProvidersResponse is the result of a GET Providers request. +type ProvidersResponse struct { + Providers RecordsArray +} + +// PeersResponse is the result of a GET Peers request. +type PeersResponse struct { + Peers RecordsArray +} + +// RecordsArray is an array of [types.Record] +type RecordsArray []types.Record + +func (r *RecordsArray) UnmarshalJSON(b []byte) error { + var tempRecords []json.RawMessage + err := json.Unmarshal(b, &tempRecords) + if err != nil { + return err + } + + for _, provBytes := range tempRecords { + var readProv types.UnknownRecord + err := json.Unmarshal(provBytes, &readProv) + if err != nil { + return err + } + + switch readProv.Schema { + case types.SchemaPeer: + var prov types.PeerRecord + err := json.Unmarshal(provBytes, &prov) + if err != nil { + return err + } + *r = append(*r, &prov) + //lint:ignore SA1019 // ignore staticcheck + case types.SchemaBitswap: + //lint:ignore SA1019 // ignore staticcheck + var prov types.BitswapRecord + err := json.Unmarshal(provBytes, &prov) + if err != nil { + return err + } + *r = append(*r, &prov) + default: + *r = append(*r, &readProv) + } + + } + return nil +} + +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +type WriteProvidersResponse struct { + ProvideResults []types.Record +} + +func (r *WriteProvidersResponse) UnmarshalJSON(b []byte) error { + var tempWPR struct{ ProvideResults []json.RawMessage } + err := json.Unmarshal(b, &tempWPR) + if err != nil { + return err + } + + for _, provBytes := range tempWPR.ProvideResults { + var rawProv types.UnknownRecord + err := json.Unmarshal(provBytes, &rawProv) + if err != nil { + return err + } + + switch rawProv.Schema { + //lint:ignore SA1019 // ignore staticcheck + case types.SchemaBitswap: + //lint:ignore SA1019 // ignore staticcheck + var prov types.WriteBitswapRecordResponse + err := json.Unmarshal(rawProv.Bytes, &prov) + if err != nil { + return err + } + r.ProvideResults = append(r.ProvideResults, &prov) + default: + r.ProvideResults = append(r.ProvideResults, &rawProv) + } + } + + return nil +} diff --git a/routing/http/types/ndjson/provider.go b/routing/http/types/ndjson/provider.go deleted file mode 100644 index 38e28df9a..000000000 --- a/routing/http/types/ndjson/provider.go +++ /dev/null @@ -1,36 +0,0 @@ -package ndjson - -import ( - "encoding/json" - "io" - - "github.com/ipfs/boxo/routing/http/types" - "github.com/ipfs/boxo/routing/http/types/iter" -) - -// NewReadProvidersResponseIter returns an iterator that reads Read Provider Records from the given reader. -func NewReadProvidersResponseIter(r io.Reader) iter.Iter[iter.Result[types.ProviderResponse]] { - jsonIter := iter.FromReaderJSON[types.UnknownProviderRecord](r) - mapFn := func(upr iter.Result[types.UnknownProviderRecord]) iter.Result[types.ProviderResponse] { - var result iter.Result[types.ProviderResponse] - if upr.Err != nil { - result.Err = upr.Err - return result - } - switch upr.Val.Schema { - case types.SchemaBitswap: - var prov types.ReadBitswapProviderRecord - err := json.Unmarshal(upr.Val.Bytes, &prov) - if err != nil { - result.Err = err - return result - } - result.Val = &prov - default: - result.Val = &upr.Val - } - return result - } - - return iter.Map[iter.Result[types.UnknownProviderRecord]](jsonIter, mapFn) -} diff --git a/routing/http/types/ndjson/records.go b/routing/http/types/ndjson/records.go new file mode 100644 index 000000000..d1a36b411 --- /dev/null +++ b/routing/http/types/ndjson/records.go @@ -0,0 +1,46 @@ +package ndjson + +import ( + "encoding/json" + "io" + + "github.com/ipfs/boxo/routing/http/types" + "github.com/ipfs/boxo/routing/http/types/iter" +) + +// NewRecordsIter returns an iterator that reads [types.Record] from the given [io.Reader]. +func NewRecordsIter(r io.Reader) iter.Iter[iter.Result[types.Record]] { + jsonIter := iter.FromReaderJSON[types.UnknownRecord](r) + mapFn := func(upr iter.Result[types.UnknownRecord]) iter.Result[types.Record] { + var result iter.Result[types.Record] + if upr.Err != nil { + result.Err = upr.Err + return result + } + switch upr.Val.Schema { + case types.SchemaPeer: + var prov types.PeerRecord + err := json.Unmarshal(upr.Val.Bytes, &prov) + if err != nil { + result.Err = err + return result + } + result.Val = &prov + //lint:ignore SA1019 // ignore staticcheck + case types.SchemaBitswap: + //lint:ignore SA1019 // ignore staticcheck + var prov types.BitswapRecord + err := json.Unmarshal(upr.Val.Bytes, &prov) + if err != nil { + result.Err = err + return result + } + result.Val = &prov + default: + result.Val = &upr.Val + } + return result + } + + return iter.Map[iter.Result[types.UnknownRecord]](jsonIter, mapFn) +} diff --git a/routing/http/types/provider.go b/routing/http/types/provider.go deleted file mode 100644 index 6e8e303f7..000000000 --- a/routing/http/types/provider.go +++ /dev/null @@ -1,17 +0,0 @@ -package types - -// WriteProviderRecord is a type that enforces structs to imlement it to avoid confusion -type WriteProviderRecord interface { - IsWriteProviderRecord() -} - -// ReadProviderRecord is a type that enforces structs to imlement it to avoid confusion -type ReadProviderRecord interface { - IsReadProviderRecord() -} - -// ProviderResponse is implemented for any ProviderResponse. It needs to have a Protocol field. -type ProviderResponse interface { - GetProtocol() string - GetSchema() string -} diff --git a/routing/http/types/provider_unknown.go b/routing/http/types/provider_unknown.go deleted file mode 100644 index 915cac481..000000000 --- a/routing/http/types/provider_unknown.go +++ /dev/null @@ -1,63 +0,0 @@ -package types - -import ( - "encoding/json" - - "github.com/ipfs/boxo/routing/http/internal/drjson" -) - -var ( - _ ReadProviderRecord = &UnknownProviderRecord{} - _ WriteProviderRecord = &UnknownProviderRecord{} - _ ProviderResponse = &UnknownProviderRecord{} -) - -// UnknownProviderRecord is used when we cannot parse the provider record using `GetProtocol` -type UnknownProviderRecord struct { - Protocol string - Schema string - Bytes []byte -} - -func (u *UnknownProviderRecord) GetProtocol() string { - return u.Protocol -} - -func (u *UnknownProviderRecord) GetSchema() string { - return u.Schema -} - -func (u *UnknownProviderRecord) IsReadProviderRecord() {} -func (u UnknownProviderRecord) IsWriteProviderRecord() {} - -func (u *UnknownProviderRecord) UnmarshalJSON(b []byte) error { - m := map[string]interface{}{} - if err := json.Unmarshal(b, &m); err != nil { - return err - } - - ps, ok := m["Protocol"].(string) - if ok { - u.Protocol = ps - } - schema, ok := m["Schema"].(string) - if ok { - u.Schema = schema - } - - u.Bytes = b - - return nil -} - -func (u UnknownProviderRecord) MarshalJSON() ([]byte, error) { - m := map[string]interface{}{} - err := json.Unmarshal(u.Bytes, &m) - if err != nil { - return nil, err - } - m["Protocol"] = u.Protocol - m["Schema"] = u.Schema - - return drjson.MarshalJSONBytes(m) -} diff --git a/routing/http/types/record.go b/routing/http/types/record.go new file mode 100644 index 000000000..4a734d5f5 --- /dev/null +++ b/routing/http/types/record.go @@ -0,0 +1,6 @@ +package types + +// Record is implemented for any record. +type Record interface { + GetSchema() string +} diff --git a/routing/http/types/provider_bitswap.go b/routing/http/types/record_bitswap.go similarity index 64% rename from routing/http/types/provider_bitswap.go rename to routing/http/types/record_bitswap.go index f0b5056e4..0780fc3eb 100644 --- a/routing/http/types/provider_bitswap.go +++ b/routing/http/types/record_bitswap.go @@ -12,14 +12,37 @@ import ( "github.com/multiformats/go-multibase" ) +// Deprecated: use the more versatile [SchemaPeer] instead. For more information, read [IPIP-417]. +// +// [IPIP-417]: https://github.com/ipfs/specs/pull/417 const SchemaBitswap = "bitswap" -var _ WriteProviderRecord = &WriteBitswapProviderRecord{} +var ( + _ Record = &BitswapRecord{} +) -// WriteBitswapProviderRecord is used when we want to add a new provider record that is using bitswap. -type WriteBitswapProviderRecord struct { - Protocol string +// Deprecated: use the more versatile [PeerRecord] instead. For more information, read [IPIP-417]. +// +// [IPIP-417]: https://github.com/ipfs/specs/pull/417 +type BitswapRecord struct { + Schema string + Protocol string + ID *peer.ID + Addrs []Multiaddr `json:",omitempty"` +} + +func (br *BitswapRecord) GetSchema() string { + return br.Schema +} + +var _ Record = &WriteBitswapRecord{} + +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +type WriteBitswapRecord struct { Schema string + Protocol string Signature string // this content must be untouched because it is signed and we need to verify it @@ -35,11 +58,13 @@ type BitswapPayload struct { Addrs []Multiaddr } -func (*WriteBitswapProviderRecord) IsWriteProviderRecord() {} +func (wr *WriteBitswapRecord) GetSchema() string { + return wr.Schema +} -type tmpBWPR WriteBitswapProviderRecord +type tmpBWPR WriteBitswapRecord -func (p *WriteBitswapProviderRecord) UnmarshalJSON(b []byte) error { +func (p *WriteBitswapRecord) UnmarshalJSON(b []byte) error { var bwp tmpBWPR err := json.Unmarshal(b, &bwp) if err != nil { @@ -54,11 +79,11 @@ func (p *WriteBitswapProviderRecord) UnmarshalJSON(b []byte) error { return json.Unmarshal(bwp.RawPayload, &p.Payload) } -func (p *WriteBitswapProviderRecord) IsSigned() bool { +func (p *WriteBitswapRecord) IsSigned() bool { return p.Signature != "" } -func (p *WriteBitswapProviderRecord) setRawPayload() error { +func (p *WriteBitswapRecord) setRawPayload() error { payloadBytes, err := drjson.MarshalJSONBytes(p.Payload) if err != nil { return fmt.Errorf("marshaling bitswap write provider payload: %w", err) @@ -69,7 +94,7 @@ func (p *WriteBitswapProviderRecord) setRawPayload() error { return nil } -func (p *WriteBitswapProviderRecord) Sign(peerID peer.ID, key crypto.PrivKey) error { +func (p *WriteBitswapRecord) Sign(peerID peer.ID, key crypto.PrivKey) error { if p.IsSigned() { return errors.New("already signed") } @@ -105,7 +130,7 @@ func (p *WriteBitswapProviderRecord) Sign(peerID peer.ID, key crypto.PrivKey) er return nil } -func (p *WriteBitswapProviderRecord) Verify() error { +func (p *WriteBitswapRecord) Verify() error { if !p.IsSigned() { return errors.New("not signed") } @@ -145,42 +170,17 @@ func (p *WriteBitswapProviderRecord) Verify() error { return nil } -var _ ProviderResponse = &WriteBitswapProviderRecordResponse{} +var _ Record = &WriteBitswapRecordResponse{} -// WriteBitswapProviderRecordResponse will be returned as a result of WriteBitswapProviderRecord -type WriteBitswapProviderRecordResponse struct { - Protocol string +// Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: +// +// [IPIP-378]: https://github.com/ipfs/specs/pull/378 +type WriteBitswapRecordResponse struct { Schema string + Protocol string AdvisoryTTL *Duration } -func (wbprr *WriteBitswapProviderRecordResponse) GetProtocol() string { - return wbprr.Protocol +func (r *WriteBitswapRecordResponse) GetSchema() string { + return r.Schema } - -func (wbprr *WriteBitswapProviderRecordResponse) GetSchema() string { - return wbprr.Schema -} - -var ( - _ ReadProviderRecord = &ReadBitswapProviderRecord{} - _ ProviderResponse = &ReadBitswapProviderRecord{} -) - -// ReadBitswapProviderRecord is a provider result with parameters for bitswap providers -type ReadBitswapProviderRecord struct { - Protocol string - Schema string - ID *peer.ID - Addrs []Multiaddr -} - -func (rbpr *ReadBitswapProviderRecord) GetProtocol() string { - return rbpr.Protocol -} - -func (rbpr *ReadBitswapProviderRecord) GetSchema() string { - return rbpr.Schema -} - -func (*ReadBitswapProviderRecord) IsReadProviderRecord() {} diff --git a/routing/http/types/record_peer.go b/routing/http/types/record_peer.go new file mode 100644 index 000000000..76bd810e0 --- /dev/null +++ b/routing/http/types/record_peer.go @@ -0,0 +1,81 @@ +package types + +import ( + "encoding/json" + + "github.com/ipfs/boxo/routing/http/internal/drjson" + "github.com/libp2p/go-libp2p/core/peer" +) + +const SchemaPeer = "peer" + +var _ Record = &PeerRecord{} + +type PeerRecord struct { + Schema string + ID *peer.ID + Addrs []Multiaddr + Protocols []string + + // Extra contains extra fields that were included in the original JSON raw + // message, except for the known ones represented by the remaining fields. + Extra map[string]json.RawMessage +} + +func (pr *PeerRecord) GetSchema() string { + return pr.Schema +} + +func (pr *PeerRecord) UnmarshalJSON(b []byte) error { + // Unmarshal all known fields and assign them. + v := struct { + Schema string + ID *peer.ID + Addrs []Multiaddr + Protocols []string + }{} + err := json.Unmarshal(b, &v) + if err != nil { + return err + } + pr.Schema = v.Schema + pr.ID = v.ID + pr.Addrs = v.Addrs + pr.Protocols = v.Protocols + + // Unmarshal everything into the Extra field and remove the + // known fields to avoid conflictual usages of the struct. + err = json.Unmarshal(b, &pr.Extra) + if err != nil { + return err + } + delete(pr.Extra, "Schema") + delete(pr.Extra, "ID") + delete(pr.Extra, "Addrs") + delete(pr.Extra, "Protocols") + + return nil +} + +func (pr PeerRecord) MarshalJSON() ([]byte, error) { + m := map[string]interface{}{} + if pr.Extra != nil { + for key, val := range pr.Extra { + m[key] = val + } + } + + // Schema and ID must always be set. + m["Schema"] = pr.Schema + m["ID"] = pr.ID + + if pr.Addrs != nil { + m["Addrs"] = pr.Addrs + } + + if pr.Protocols != nil { + m["Protocols"] = pr.Protocols + } + + return drjson.MarshalJSONBytes(m) +} diff --git a/routing/http/types/record_unknown.go b/routing/http/types/record_unknown.go new file mode 100644 index 000000000..9b2f6f960 --- /dev/null +++ b/routing/http/types/record_unknown.go @@ -0,0 +1,47 @@ +package types + +import ( + "encoding/json" + + "github.com/ipfs/boxo/routing/http/internal/drjson" +) + +var _ Record = &UnknownRecord{} + +type UnknownRecord struct { + Schema string + + // Bytes contains the raw JSON bytes that were used to unmarshal this record. + // This value can be used, for example, to unmarshal de record into a different + // type if Schema is of a known value. + Bytes []byte +} + +func (ur *UnknownRecord) GetSchema() string { + return ur.Schema +} + +func (ur *UnknownRecord) UnmarshalJSON(b []byte) error { + v := struct { + Schema string + }{} + err := json.Unmarshal(b, &v) + if err != nil { + return err + } + ur.Schema = v.Schema + ur.Bytes = b + return nil +} + +func (ur UnknownRecord) MarshalJSON() ([]byte, error) { + m := map[string]interface{}{} + if ur.Bytes != nil { + err := json.Unmarshal(ur.Bytes, &m) + if err != nil { + return nil, err + } + } + m["Schema"] = ur.Schema + return drjson.MarshalJSONBytes(m) +}