Skip to content

Commit

Permalink
Data Transfer V2 Integration
Browse files Browse the repository at this point in the history
Voucher refactor integration (#707)

* refactor(storagemarket): update storagemarket

* refactor(retrievalmarket): use new voucher system

refactor for predictability and correctness using new voucher system

* chore(deps): update go-data-transfer v2

* style(lint): prep for pr

* docs(retrievalmarket): add comments

* chore(deps): update go-statemachine

* style(imports): fix imports

chore(retrievalmarket): remove old types (#712)

feat(ipld): bindnode support for all voucher types (#713)

* feat(ipld): new data-transfer ipld vouchers + bindnode

* feat(ipld): simplify ipldutils API

* feat(ipld): use new bindnode registry in go-ipld-prime

Ref: ipld/go-ipld-prime#437

feat(deps): update data transfer 61f0756c

feat(deps): update data transfer and other deps

update to master data transfer with libp2p v0.22.0 plus associated other deps

Update retrievalmarket/types.go

Update storagemarket/types.go

chore(deps): update to rc candidate

chore(dtutils): update for new data transfer configuration interface
  • Loading branch information
hannahhoward committed Feb 11, 2023
1 parent a96e387 commit ca5e771
Show file tree
Hide file tree
Showing 81 changed files with 2,637 additions and 5,964 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ orbs:
executors:
golang:
docker:
- image: cimg/go:1.17.9
- image: cimg/go:1.18.7-node
resource_class: large

commands:
Expand Down
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ import (
ipld "github.com/ipfs/go-ipld-format"
"github.com/stretchr/testify/assert"

datatransfer "github.com/filecoin-project/go-data-transfer"
datatransfer "github.com/filecoin-project/go-data-transfer/v2"

"github.com/filecoin-project/go-fil-markets/filestore/file"
)
Expand Down
60 changes: 0 additions & 60 deletions discovery/impl/local_test.go
Original file line number Diff line number Diff line change
@@ -1,23 +1,17 @@
package discoveryimpl_test

import (
"bytes"
"context"
"testing"
"time"

"github.com/ipfs/go-datastore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/filecoin-project/go-address"

discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl"
"github.com/filecoin-project/go-fil-markets/discovery/migrations"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
retrievalmigrations "github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations"
"github.com/filecoin-project/go-fil-markets/shared_testutil"
)

Expand Down Expand Up @@ -70,57 +64,3 @@ func TestLocal_AddPeer(t *testing.T) {
})
}
}

func TestLocalMigrations(t *testing.T) {
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
ds := datastore.NewMapDatastore()

peers := shared_testutil.GeneratePeers(4)
pieceCIDs := shared_testutil.GenerateCids(4)
payloadCids := shared_testutil.GenerateCids(2)
for i, c := range payloadCids {
rps := migrations.RetrievalPeers0{
Peers: []retrievalmigrations.RetrievalPeer0{
{
Address: address.TestAddress,
ID: peers[i*2],
PieceCID: &pieceCIDs[i*2],
},
{
Address: address.TestAddress2,
ID: peers[i*2+1],
PieceCID: &pieceCIDs[i*2+1],
},
},
}
buf := new(bytes.Buffer)
err := rps.MarshalCBOR(buf)
require.NoError(t, err)
err = ds.Put(ctx, dshelp.MultihashToDsKey(c.Hash()), buf.Bytes())
require.NoError(t, err)
}

l, err := discoveryimpl.NewLocal(ds)
require.NoError(t, err)
shared_testutil.StartAndWaitForReady(ctx, t, l)

for i, c := range payloadCids {
expectedPeers := []retrievalmarket.RetrievalPeer{
{
Address: address.TestAddress,
ID: peers[i*2],
PieceCID: &pieceCIDs[i*2],
},
{
Address: address.TestAddress2,
ID: peers[i*2+1],
PieceCID: &pieceCIDs[i*2+1],
},
}
peers, err := l.GetPeers(c)
require.NoError(t, err)
require.Equal(t, expectedPeers, peers)
}
}
31 changes: 1 addition & 30 deletions discovery/migrations/migrations.go
Original file line number Diff line number Diff line change
@@ -1,37 +1,8 @@
package migrations

import (
versioning "github.com/filecoin-project/go-ds-versioning/pkg"
"github.com/filecoin-project/go-ds-versioning/pkg/versioned"

"github.com/filecoin-project/go-fil-markets/discovery"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations"
)

//go:generate cbor-gen-for RetrievalPeers0

// RetrievalPeers0 is version 0 of RetrievalPeers
type RetrievalPeers0 struct {
Peers []migrations.RetrievalPeer0
}

// MigrateRetrievalPeers0To1 migrates a tuple encoded list of retrieval peers to a map encoded list
func MigrateRetrievalPeers0To1(oldRps *RetrievalPeers0) (*discovery.RetrievalPeers, error) {
peers := make([]retrievalmarket.RetrievalPeer, 0, len(oldRps.Peers))
for _, oldRp := range oldRps.Peers {
peers = append(peers, retrievalmarket.RetrievalPeer{
Address: oldRp.Address,
ID: oldRp.ID,
PieceCID: oldRp.PieceCID,
})
}
return &discovery.RetrievalPeers{
Peers: peers,
}, nil
}

// RetrievalPeersMigrations are migrations for the store local discovery list of peers we can retrieve from
var RetrievalPeersMigrations = versioned.BuilderList{
versioned.NewVersionedBuilder(MigrateRetrievalPeers0To1, versioning.VersionKey("1")),
}
var RetrievalPeersMigrations = versioned.BuilderList{}
105 changes: 0 additions & 105 deletions discovery/migrations/migrations_cbor_gen.go

This file was deleted.

14 changes: 9 additions & 5 deletions docs/retrievalprovider.mmd
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,18 @@ stateDiagram-v2
state "DealStatusCancelled" as DealStatusCancelled
DealStatusUnsealing : On entry runs UnsealData
DealStatusUnsealed : On entry runs UnpauseDeal
DealStatusFundsNeededUnseal : On entry runs TrackTransfer
DealStatusFundsNeededUnseal : On entry runs UpdateFunding
DealStatusFailing : On entry runs CancelDeal
DealStatusFundsNeeded : On entry runs UpdateFunding
DealStatusFundsNeededLastPayment : On entry runs UpdateFunding
DealStatusCompleting : On entry runs CleanupDeal
DealStatusCancelling : On entry runs CancelDeal
[*] --> DealStatusNew
note right of DealStatusNew
The following events are not shown cause they can trigger from any state.
ProviderEventProcessPayment - does not transition state
ProviderEventComplete - transitions state to DealStatusCompleting
ProviderEventDataTransferError - transitions state to DealStatusErrored
ProviderEventMultiStoreError - transitions state to DealStatusErrored
ProviderEventClientCancelled - transitions state to DealStatusCancelling
Expand All @@ -35,20 +39,20 @@ stateDiagram-v2
DealStatusUnsealing --> DealStatusUnsealed : ProviderEventUnsealComplete
DealStatusUnsealed --> DealStatusOngoing : ProviderEventBlockSent
DealStatusOngoing --> DealStatusOngoing : ProviderEventBlockSent
DealStatusOngoing --> DealStatusBlocksComplete : ProviderEventBlocksCompleted
DealStatusNew --> DealStatusFundsNeededUnseal : ProviderEventPaymentRequested
DealStatusUnsealed --> DealStatusFundsNeeded : ProviderEventPaymentRequested
DealStatusOngoing --> DealStatusFundsNeeded : ProviderEventPaymentRequested
DealStatusBlocksComplete --> DealStatusFundsNeededLastPayment : ProviderEventPaymentRequested
DealStatusUnsealed --> DealStatusFundsNeededLastPayment : ProviderEventLastPaymentRequested
DealStatusOngoing --> DealStatusFundsNeededLastPayment : ProviderEventLastPaymentRequested
DealStatusFundsNeededUnseal --> DealStatusFailing : ProviderEventSaveVoucherFailed
DealStatusFundsNeeded --> DealStatusFailing : ProviderEventSaveVoucherFailed
DealStatusFundsNeededLastPayment --> DealStatusFailing : ProviderEventSaveVoucherFailed
DealStatusFundsNeededUnseal --> DealStatusFundsNeededUnseal : ProviderEventPartialPaymentReceived
DealStatusFundsNeeded --> DealStatusFundsNeeded : ProviderEventPartialPaymentReceived
DealStatusFundsNeededLastPayment --> DealStatusFundsNeededLastPayment : ProviderEventPartialPaymentReceived
DealStatusFundsNeededUnseal --> DealStatusUnsealing : ProviderEventPaymentReceived
DealStatusFundsNeeded --> DealStatusOngoing : ProviderEventPaymentReceived
DealStatusFundsNeededLastPayment --> DealStatusFinalizing : ProviderEventPaymentReceived
DealStatusBlocksComplete --> DealStatusCompleting : ProviderEventComplete
DealStatusFinalizing --> DealStatusCompleting : ProviderEventComplete
DealStatusCompleting --> DealStatusCompleted : ProviderEventCleanupComplete
DealStatusFailing --> DealStatusErrored : ProviderEventCancelComplete
DealStatusCancelling --> DealStatusCancelled : ProviderEventCancelComplete
Expand Down
Binary file modified docs/retrievalprovider.mmd.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
6 changes: 3 additions & 3 deletions docs/retrievalprovider.mmd.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
13 changes: 6 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
module github.com/filecoin-project/go-fil-markets

go 1.17
go 1.18

require (
github.com/filecoin-project/dagstore v0.5.2
github.com/filecoin-project/go-address v0.0.6
github.com/filecoin-project/go-cbor-util v0.0.1
github.com/filecoin-project/go-commp-utils v0.1.3
github.com/filecoin-project/go-data-transfer v1.15.2
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc2.0.20230211023954-5f99b068a7f2
github.com/filecoin-project/go-ds-versioning v0.1.2
github.com/filecoin-project/go-fil-commcid v0.1.0
github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
github.com/filecoin-project/go-padreader v0.0.1
github.com/filecoin-project/go-state-types v0.1.12
github.com/filecoin-project/go-statemachine v1.0.2
github.com/filecoin-project/go-statemachine v1.0.3
github.com/filecoin-project/go-statestore v0.2.0
github.com/filecoin-project/specs-actors v0.9.13
github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1
Expand All @@ -25,7 +25,7 @@ require (
github.com/ipfs/go-cidutil v0.1.0
github.com/ipfs/go-datastore v0.6.0
github.com/ipfs/go-filestore v1.2.0
github.com/ipfs/go-graphsync v0.13.2
github.com/ipfs/go-graphsync v0.14.0
github.com/ipfs/go-ipfs-blockstore v1.2.0
github.com/ipfs/go-ipfs-blocksutil v0.0.1
github.com/ipfs/go-ipfs-chunker v0.0.5
Expand All @@ -35,7 +35,7 @@ require (
github.com/ipfs/go-ipld-cbor v0.0.6
github.com/ipfs/go-ipld-format v0.4.0
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipfs/go-merkledag v0.6.0
github.com/ipfs/go-merkledag v0.8.1
github.com/ipfs/go-unixfs v0.4.0
github.com/ipld/go-car v0.4.0
github.com/ipld/go-car/v2 v2.5.0
Expand Down Expand Up @@ -105,7 +105,6 @@ require (
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
github.com/libp2p/go-libp2p-core v0.20.1 // indirect
github.com/libp2p/go-msgio v0.2.0 // indirect
github.com/libp2p/go-nat v0.1.0 // indirect
github.com/libp2p/go-netroute v0.2.0 // indirect
Expand All @@ -132,9 +131,9 @@ require (
github.com/twmb/murmur3 v1.1.6 // indirect
github.com/urfave/cli/v2 v2.16.3 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.opentelemetry.io/otel v1.10.0 // indirect
go.opentelemetry.io/otel/sdk v1.10.0 // indirect
go.opentelemetry.io/otel/trace v1.10.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
Expand Down
Loading

0 comments on commit ca5e771

Please sign in to comment.