From 7b745c3c28199ca5a4c175aeebecd0e30b88b6f0 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Fri, 17 Mar 2023 16:42:04 +0100 Subject: [PATCH] feat: add local index directory Internally this is still refered to as the piece directory Co-authored-by: dirkmc Co-authored-by: Anton Evangelatov --- .circleci/config.yml | 18 +- Makefile | 2 +- api/api.go | 12 +- api/proxy_gen.go | 81 +- build/openrpc/boost.json.gz | Bin 5490 -> 5353 bytes cmd/boost/retrieve_cmd.go | 9 +- cmd/boostci/utils_cmd.go | 1 - cmd/boostd/index.go | 43 +- cmd/boostd/main.go | 10 +- cmd/boostd/piecedir.go | 100 ++ cmd/boostd/pieces.go | 290 --- cmd/booster-bitswap/init.go | 11 +- cmd/booster-bitswap/run.go | 81 +- cmd/booster-http/http_test.go | 38 +- cmd/booster-http/mocks/mock_booster_http.go | 31 +- cmd/booster-http/run.go | 77 +- cmd/booster-http/server.go | 56 +- cmd/migrate-lid/Makefile | 17 + cmd/migrate-lid/main.go | 63 + cmd/migrate-lid/migrate_lid.go | 868 +++++++++ documentation/en/api-v1-methods.md | 127 +- extern/boostd-data/Makefile | 17 + extern/boostd-data/client/client.go | 171 +- extern/boostd-data/cmd/main.go | 43 + extern/boostd-data/cmd/run.go | 158 ++ extern/boostd-data/couchbase/db.go | 1221 +++++++++++-- extern/boostd-data/couchbase/encode.go | 80 + extern/boostd-data/couchbase/encode_test.go | 79 + extern/boostd-data/couchbase/service.go | 396 +++-- extern/boostd-data/couchbase/shard.go | 64 + extern/boostd-data/couchbase/shard_test.go | 218 +++ extern/boostd-data/go.mod | 109 +- extern/boostd-data/go.sum | 1560 ++++++++++++++++- extern/boostd-data/ldb/db.go | 504 +++++- extern/boostd-data/ldb/service.go | 496 +++++- extern/boostd-data/main.go | 72 - extern/boostd-data/model/model.go | 74 +- extern/boostd-data/model/model_test.go | 49 + extern/boostd-data/svc/couchbase_test.go | 105 -- .../svc/setup_couchbase_test_util.go | 255 +++ extern/boostd-data/svc/svc.go | 87 +- extern/boostd-data/svc/svc_test.go | 503 ++++-- extern/boostd-data/svc/types/types.go | 50 + go.mod | 35 +- go.sum | 51 +- gql/resolver.go | 88 +- gql/resolver_funds.go | 10 +- gql/resolver_piece.go | 291 ++- gql/resolver_proplog.go | 9 +- gql/schema.graphql | 20 + gql/util.go | 16 + indexprovider/wrapper.go | 253 +-- markets/storageadapter/api.go | 10 +- .../ondealsectorcommitted_test.go | 12 +- metrics/metrics.go | 6 + node/builder.go | 30 +- node/config/def.go | 14 + node/config/doc_gen.go | 85 + node/config/types.go | 45 +- node/impl/boost.go | 49 +- node/impl/boost_legacy.go | 50 - node/modules/dtypes/storage.go | 1 - node/modules/graphsync.go | 15 +- node/modules/piecedirectory.go | 285 +++ node/modules/provider_piece_store.go | 24 + node/modules/storageminer.go | 20 +- piecedirectory/doctor.go | 149 ++ piecedirectory/doctor_test.go | 329 ++++ piecedirectory/piecedirectory.go | 636 +++++++ piecedirectory/piecedirectory_test.go | 365 ++++ piecedirectory/test_util.go | 117 ++ piecedirectory/types/mocks/piecedirectory.go | 453 +++++ piecedirectory/types/types.go | 60 + react/src/App.js | 6 +- react/src/DealPublish.js | 6 +- react/src/Deals.js | 6 +- react/src/Epoch.js | 2 +- react/src/Funds.js | 8 +- react/src/Info.css | 11 +- react/src/Info.js | 9 + react/src/Inspect.css | 20 +- react/src/Inspect.js | 253 ++- react/src/LegacyDealDetail.js | 2 +- react/src/LegacyDeals.js | 2 +- react/src/PieceDirectory.css | 79 + react/src/PieceDirectory.js | 279 +++ react/src/ProposalLogs.js | 4 +- react/src/RetrievalLogDetail.js | 2 +- react/src/RetrievalLogs.js | 4 +- react/src/SealingPipeline.js | 4 +- react/src/StorageSpace.js | 8 +- react/src/gql.js | 35 + storagemarket/deal_execution.go | 56 +- storagemarket/provider.go | 25 +- storagemarket/provider_test.go | 41 +- 95 files changed, 10551 insertions(+), 2085 deletions(-) create mode 100644 cmd/boostd/piecedir.go delete mode 100644 cmd/boostd/pieces.go create mode 100644 cmd/migrate-lid/Makefile create mode 100644 cmd/migrate-lid/main.go create mode 100644 cmd/migrate-lid/migrate_lid.go create mode 100644 extern/boostd-data/Makefile create mode 100644 extern/boostd-data/cmd/main.go create mode 100644 extern/boostd-data/cmd/run.go create mode 100644 extern/boostd-data/couchbase/encode.go create mode 100644 extern/boostd-data/couchbase/encode_test.go create mode 100644 extern/boostd-data/couchbase/shard.go create mode 100644 extern/boostd-data/couchbase/shard_test.go delete mode 100644 extern/boostd-data/main.go create mode 100644 extern/boostd-data/model/model_test.go delete mode 100644 extern/boostd-data/svc/couchbase_test.go create mode 100644 extern/boostd-data/svc/setup_couchbase_test_util.go create mode 100644 extern/boostd-data/svc/types/types.go create mode 100644 gql/util.go create mode 100644 node/modules/piecedirectory.go create mode 100644 node/modules/provider_piece_store.go create mode 100644 piecedirectory/doctor.go create mode 100644 piecedirectory/doctor_test.go create mode 100644 piecedirectory/piecedirectory.go create mode 100644 piecedirectory/piecedirectory_test.go create mode 100644 piecedirectory/test_util.go create mode 100644 piecedirectory/types/mocks/piecedirectory.go create mode 100644 piecedirectory/types/types.go create mode 100644 react/src/PieceDirectory.css create mode 100644 react/src/PieceDirectory.js diff --git a/.circleci/config.yml b/.circleci/config.yml index 67005a095..12bba03c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2.1 orbs: - go: gotest/tools@0.0.13 + go: gotest/tools@0.0.14 executors: golang: @@ -98,6 +98,10 @@ jobs: type: string default: "./..." description: Import paths of packages to be tested. + cwd: + type: string + default: "." + description: Directory in which to run the test command suite: type: string default: unit @@ -110,9 +114,16 @@ jobs: command: make boostci no_output_timeout: 30m - download-params + # Note: setup_remote_docker takes about 10s and it's only needed for + # local index directory tests so it may be worth refactoring so that it's + # only included for local index directory + - setup_remote_docker: + version: 20.10.18 + docker_layer_caching: true - run: name: go test command: | + cd << parameters.cwd >> go test \ << parameters.go-test-flags >> \ << parameters.target >> @@ -310,3 +321,8 @@ workflows: name: test-all suite: all target: "`go list ./... | grep -v boost/itests`" + + - test: + name: local index directory + suite: all + cwd: "./extern/boostd-data" diff --git a/Makefile b/Makefile index 0017a1134..40e8930d7 100644 --- a/Makefile +++ b/Makefile @@ -226,7 +226,7 @@ $(lotus_src_dir): git clone --depth 1 --branch $(lotus_version) https://github.com/filecoin-project/lotus $@ update/lotus: $(lotus_src_dir) cd $(lotus_src_dir) && git pull -.PHONY: update/lotus +.PHONY: update/lotus docker/lotus-all-in-one: info/lotus-all-in-one | $(lotus_src_dir) cd $(lotus_src_dir) && $(docker_build_cmd) -f Dockerfile --target lotus-all-in-one \ diff --git a/api/api.go b/api/api.go index 6172ef613..71b19350c 100644 --- a/api/api.go +++ b/api/api.go @@ -6,7 +6,6 @@ import ( smtypes "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/go-address" datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" @@ -35,6 +34,7 @@ type Boost interface { // MethodGroup: Boost BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin + BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error) //perm:admin BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string) (*ProviderDealRejectionInfo, error) //perm:admin BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error) //perm:admin BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error) //perm:admin @@ -54,6 +54,10 @@ type Boost interface { BlockstoreHas(ctx context.Context, c cid.Cid) (bool, error) //perm:read BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) //perm:read + // MethodGroup: PieceDirectory + PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error //perm:admin + PdMarkIndexErrored(ctx context.Context, piececid cid.Cid, err string) error //perm:admin + // RuntimeSubsystems returns the subsystems that are enabled // in this instance. RuntimeSubsystems(ctx context.Context) (lapi.MinerSubsystems, error) //perm:read @@ -73,12 +77,6 @@ type Boost interface { MarketPendingDeals(ctx context.Context) (lapi.PendingDealInfo, error) //perm:write SectorsRefs(context.Context) (map[string][]lapi.SealedRef, error) //perm:read - PiecesListPieces(ctx context.Context) ([]cid.Cid, error) //perm:read - PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) //perm:read - PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read - PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) //perm:read - PiecesGetMaxOffset(ctx context.Context, pieceCid cid.Cid) (uint64, error) //perm:read - // MethodGroup: Actor ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 2f2688cec..2aef61f8b 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -10,7 +10,6 @@ import ( smtypes "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/go-address" datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-jsonrpc/auth" @@ -68,6 +67,8 @@ type BoostStruct struct { BoostIndexerAnnounceAllDeals func(p0 context.Context) error `perm:"admin"` + BoostIndexerListMultihashes func(p0 context.Context, p1 cid.Cid) ([]multihash.Multihash, error) `perm:"admin"` + BoostMakeDeal func(p0 context.Context, p1 smtypes.DealParams) (*ProviderDealRejectionInfo, error) `perm:"write"` BoostOfflineDealWithData func(p0 context.Context, p1 uuid.UUID, p2 string) (*ProviderDealRejectionInfo, error) `perm:"admin"` @@ -126,15 +127,9 @@ type BoostStruct struct { OnlineBackup func(p0 context.Context, p1 string) error `perm:"admin"` - PiecesGetCIDInfo func(p0 context.Context, p1 cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"` - - PiecesGetMaxOffset func(p0 context.Context, p1 cid.Cid) (uint64, error) `perm:"read"` - - PiecesGetPieceInfo func(p0 context.Context, p1 cid.Cid) (*piecestore.PieceInfo, error) `perm:"read"` + PdBuildIndexForPieceCid func(p0 context.Context, p1 cid.Cid) error `perm:"admin"` - PiecesListCidInfos func(p0 context.Context) ([]cid.Cid, error) `perm:"read"` - - PiecesListPieces func(p0 context.Context) ([]cid.Cid, error) `perm:"read"` + PdMarkIndexErrored func(p0 context.Context, p1 cid.Cid, p2 string) error `perm:"admin"` RuntimeSubsystems func(p0 context.Context) (lapi.MinerSubsystems, error) `perm:"read"` @@ -441,6 +436,17 @@ func (s *BoostStub) BoostIndexerAnnounceAllDeals(p0 context.Context) error { return ErrNotSupported } +func (s *BoostStruct) BoostIndexerListMultihashes(p0 context.Context, p1 cid.Cid) ([]multihash.Multihash, error) { + if s.Internal.BoostIndexerListMultihashes == nil { + return *new([]multihash.Multihash), ErrNotSupported + } + return s.Internal.BoostIndexerListMultihashes(p0, p1) +} + +func (s *BoostStub) BoostIndexerListMultihashes(p0 context.Context, p1 cid.Cid) ([]multihash.Multihash, error) { + return *new([]multihash.Multihash), ErrNotSupported +} + func (s *BoostStruct) BoostMakeDeal(p0 context.Context, p1 smtypes.DealParams) (*ProviderDealRejectionInfo, error) { if s.Internal.BoostMakeDeal == nil { return nil, ErrNotSupported @@ -760,59 +766,26 @@ func (s *BoostStub) OnlineBackup(p0 context.Context, p1 string) error { return ErrNotSupported } -func (s *BoostStruct) PiecesGetCIDInfo(p0 context.Context, p1 cid.Cid) (*piecestore.CIDInfo, error) { - if s.Internal.PiecesGetCIDInfo == nil { - return nil, ErrNotSupported - } - return s.Internal.PiecesGetCIDInfo(p0, p1) -} - -func (s *BoostStub) PiecesGetCIDInfo(p0 context.Context, p1 cid.Cid) (*piecestore.CIDInfo, error) { - return nil, ErrNotSupported -} - -func (s *BoostStruct) PiecesGetMaxOffset(p0 context.Context, p1 cid.Cid) (uint64, error) { - if s.Internal.PiecesGetMaxOffset == nil { - return 0, ErrNotSupported - } - return s.Internal.PiecesGetMaxOffset(p0, p1) -} - -func (s *BoostStub) PiecesGetMaxOffset(p0 context.Context, p1 cid.Cid) (uint64, error) { - return 0, ErrNotSupported -} - -func (s *BoostStruct) PiecesGetPieceInfo(p0 context.Context, p1 cid.Cid) (*piecestore.PieceInfo, error) { - if s.Internal.PiecesGetPieceInfo == nil { - return nil, ErrNotSupported - } - return s.Internal.PiecesGetPieceInfo(p0, p1) -} - -func (s *BoostStub) PiecesGetPieceInfo(p0 context.Context, p1 cid.Cid) (*piecestore.PieceInfo, error) { - return nil, ErrNotSupported -} - -func (s *BoostStruct) PiecesListCidInfos(p0 context.Context) ([]cid.Cid, error) { - if s.Internal.PiecesListCidInfos == nil { - return *new([]cid.Cid), ErrNotSupported +func (s *BoostStruct) PdBuildIndexForPieceCid(p0 context.Context, p1 cid.Cid) error { + if s.Internal.PdBuildIndexForPieceCid == nil { + return ErrNotSupported } - return s.Internal.PiecesListCidInfos(p0) + return s.Internal.PdBuildIndexForPieceCid(p0, p1) } -func (s *BoostStub) PiecesListCidInfos(p0 context.Context) ([]cid.Cid, error) { - return *new([]cid.Cid), ErrNotSupported +func (s *BoostStub) PdBuildIndexForPieceCid(p0 context.Context, p1 cid.Cid) error { + return ErrNotSupported } -func (s *BoostStruct) PiecesListPieces(p0 context.Context) ([]cid.Cid, error) { - if s.Internal.PiecesListPieces == nil { - return *new([]cid.Cid), ErrNotSupported +func (s *BoostStruct) PdMarkIndexErrored(p0 context.Context, p1 cid.Cid, p2 string) error { + if s.Internal.PdMarkIndexErrored == nil { + return ErrNotSupported } - return s.Internal.PiecesListPieces(p0) + return s.Internal.PdMarkIndexErrored(p0, p1, p2) } -func (s *BoostStub) PiecesListPieces(p0 context.Context) ([]cid.Cid, error) { - return *new([]cid.Cid), ErrNotSupported +func (s *BoostStub) PdMarkIndexErrored(p0 context.Context, p1 cid.Cid, p2 string) error { + return ErrNotSupported } func (s *BoostStruct) RuntimeSubsystems(p0 context.Context) (lapi.MinerSubsystems, error) { diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index 675a2100e8a317cd92f3356c81a16867ed04e74d..93e553911061689791065aa632cc11c11a6754de 100644 GIT binary patch literal 5353 zcmVHM_ zKYP$Aq88N8^ecz@M7y3|YUk65-qG)%N05Vgk8!vE(lh${rlZ@Ct{ju-XMaBE7}%`m zTVz4QLAZNvP{*4={x|vo^^R^rVtB}<0ids6zb>6Gc;RD1dq%Vu7mh>djC#I7pKF=U z&)lA7IGDnF+Pw_^?PzXa1E%RgLb~U{zt6R2V2--mejN>Zp5r~&9=_l&6fHE;onQ(* z46H1E+AqIo7+FEfFVut7_ptT?!|xfG&pX;1&->`mneV!eN1@4jY}j96C;3aBxjlyW z^DlVw1%LheRqyD;w{769^)vm`3VKij9@MZywKb#~z-!HLY#U-qw1wkobcKi(OnO)E z=q~Vp9ZZ?;vB{sWU%C!Ke1^lHQ;XyFbIXvK!W2gAi?M<>V59->f$dt5!P6JL{$M!L zZ#t|ye;u%nvh2B7Q- z@3pyMjfihi_Ro7I13-sS=lFaeZn$8ctI=Ve`4g zRGXL(Q?x(=BELkY<}n&19AlsnXf0j_ZRQ9sgLT3*lC99xPzhi=Jx$n;OtQuvRNT zR^2q;J?O94yn)4fF#iDz2!`K@xf-vBU^u4pA-RRN` zugAU*M}xbOzDckwp&r7^9pq6!W4XDFA1jE^|6;56$LwNd;D4>)|NioeUKhV7DpHO| z3L>StVUX)yE?N+#%5p_+3G`|Ulv2gtN{Pdi5(wvYX7Fr3_DO#*JTSZDpNpNe)JJxtR?gmfIHWO$lsRKM3&8EeTG? z5=p>1zXnUrF0Ub>p0l2oizLn>W@L@t`#d^@Pernnu_Kp>3|J~&d6#wp0K?^$*e3KbEDFq$y#E7=Yc0KLMvH>Q8`d znEcbow)B69^nW7Dec+@3L4Gn6$Q=SpM%})T%G{o0d=HgU;H2_K~TdsZDA}zw{s*z&T}IdcFQj{^wJ#cgFwx!J1eIVop2EJ9h}M z*n3l-ng4S+dCfjIVQ6;(h&&bPtlt|BI{NR>W7e?ASv8#h05Sb6%6;wV(Yz&}U4c2Y zf?1nU;L$g(W2_k1ey`ur-(WNSU`8DeEaB8c2Ao1K{88^EpkF$c1t|0wCj5s0k<8;w z^n8Yv7zE7ppnu|2HR$NC0R{S5KId$+>F5*k+hSoM4AUc>Lg=wc*879k!(Sce-|v3h z|M>Nf|K7p(|Do=?@j3B6|Lel|_x<~ex3j@}^2T}h>E8Qrb@%Ik=tA#s>R_+Eb{yK= zQcdvO@v+Gy>J-pbtl&bW)zLqB0Fwpuw!_c12!_Xpmic8GiktjAb@YHdMgum{D}XQ? z9)oc-Vgxhw=N2L>PL)i3b@U(|nSDR)=)Wx%1k#AQt{`60RrKw01&v$RL5#Ng$*)CR z^2YP>=wCz1ITrZT(L4Ibi#3Jh8X6GYF`;qV(cc0>St|tZwuF6-vEyR{esU($kCWi( zrYy5N9Vg3z$nW!HW=$Ye!ye)HO?qOgwLj-Qm+iM%J6U&$bW$DHjHGH(!B(uMs=hU?RSWil<31w7 zO=~AX2tHwCs0x9suBx{5TRZ3>vKq5ahuB6eo%eE!F*F!Ap-K=;M@Ht0L~~B53cw_+ z1QCcxr>Rg=Rxzpci8QDZZzYs*)wK+DsAy*=R_H1?B+Zb@;#ES}k$F;6Tc``CSg%O- zu=4y_@rc(Gb=yy6*~XDOd$DGE9-5Kz6m1r$dGZ2C<+;c^y7q#1HtX$+2i0aT|I@OU z+aN?6glK~hZ4jajLbO4MHVDxMA=)5B8-!?s5N!~m4MLO$Ax6EzgO0v!uj?idM17?> zGNCKR?GV1q&$VYSl8)ImWaB^%R?C@>Uf5uiAu)#ir_SW&ql!=ZnH%?tKB>h0X+6-# z^Fa6GDn5cIcnxjm4t~Z20t=d#j=PT5YIciU<_krhQ7yNs^-Wser1edHBHyF|KjL$+ zz<-|~QDW$eM4qT!UI2<(wPteV0Ke2I%m?_3nyMI6=Y|1Yy2bDs0+PsV4z*>~SYZrQ zOkTk&luq&e#I8oasIJ5WoA4fb5$l{=7F)qO=W->GgoksB^YbN;8p+U?WBJ&XAlnT3 ztI?pSYC^bFSBTJfQxxt%nHfMT#YzR|AWQvW@!JXyi;9k_R(6`DqilEP|7kn(LW&Zc zcddKOH#+B=IzO@hBfkfAa&$C2Es%X5z%8s90F1gmR{Sq|UvuHV#0c8xUmN{vqknDm zuZ{k-(Z4qO*GB)^=wBQCYomWv+>E25f_xzZvSs@>qARY4t9u**D5WI~I+{C(%XRGH zeh6OPoF^h485;@1zpZz$k9r4NT6G6cv(8B?84p4-jwPiG`G&xyg9%%=Py)nRjWu>h z%HfF%n3mm1Q4Y7Eiz<``u$P*p?F9*0qCJ+>n-i^3=&&?IvPo#o5o{9b9?3O>d`qm4 zLaZl}0&(dT!p#a;0k|f}RRDG`^eRTbCD=zH*i%VGH^CA$C0ujB8Y9-di&hcpPa%t< zrRxk`AN3?94^r#-8Q+Dm$%bfK46p(E76aY~jZ25n3gky2+`i}8G2a;qXSjdD9}XU51N-?GKQu*F!i>z*ToW`dt35SwBQfo3bP^Bs1m5H&mPhtv@CKkEOJ`3 zm7Ji3hGM}b5F26-^^kCV=8&M@vc{pX#<*9sX+49qiKw#(jEo7kLE>^wPz*j-Cw~#P zd#Cv}!2Fa0%&n}r8uulM03U#N3+W}m2IS|8;^Q)vM1>5BOj4MD0vfjIMvcR>0WO4I zmwz6Xsa4r)cadsE9M(y#@ko+`5lhW@zo@>NNP>{)3omLCN<>v9kp$hW zw#Xp8AGIMEc3xMA8>uLWsGh>bnnubXIP{-X4^yzVA*Or=_}v1dLGIxM8y*BCD49}C za_7J+Rj-h}Uaj;+a@18NXKI3KO1Hxh4M&pvlYm7gHshtJSVkSmkcLoFGca|iy~JHj z18&Au%^~|JC)L~Xe}7~7zo&m%{-5T`9!pYx0v5eOgeGH26*s3OKx8@K7I0BF+46D) zFouPP79_7I%nFyir&5}@d~AEl6PIBvi0cua(-=hFxbF;bGWOigK zn4t+;s8-M`wdTTWw_*wq{TeU3pvYBX`lZ4$;{if_iMb_9V-aXS!T7fI|s8zB(ZM7(r0?#T{CDKZmU23EVsA>swkr7g1 zK~)uqC=Qk+q}XJsSxx=00tKg%gJ)YGc$cHeGAObzK{kgVSv#4Nnp(g><$!V+FDQLu z&B%db{l>>Fi>EMLLBv6&k6P>jtStwa&F*zDSBObEi_1C&_!@kddFZh{5jtb}#FMIN zCZbdork^2u3;Q_|cdwtG&&8*MS0dj_Fes20(M?%fpEb}*?E7g)|821#kVfI6D~OkL z6@1&=Bm&AkHgE%?yPfu*915%%H<;PVgl+_f8bt`C;e<=y^Kx`?1((2BPd$e^hGRwd zF4gr;Q3s7Xjtr7Fhiv;5E$r%qaG< zGblneH{+5+!22?1*Qqj*b3-q?vdzusnnVk6iTQnsN$g}WfQ}gVYgQqm7F!j zOIh_Yo4uamEs{Y`5zD;DNfX@O#60%`8xZv);Q%shu*Xph2MrA3=%*b-t2(ZZhlLe# zX1muwn3dK*;L1dF(qBaZlovs$p$Rew@4diZTjt=<%N(?#=IVGPxx*?bLw*5_Ti>lF zVUj&%f~B->LK4WX5+s9=TyX3!MW;y4vDLa%0 zySuCqAljB3I&|J|OAb9XNpvD9Idtt~iflOZ=VVQI8bBpMl*T@by&d7ez(JT$fDPFFf}6!i}4l5PzR8mSK$RQ=ya8Z$|5*@|tWCQolKmy)`b*E+! zSkSzNiz2Q{z%AgX0^pU{Dm{w5B)=7)nw#*!G;8~?uH1n&aMCT5z7wsl|BO~2Y#GINA`6KT(fl{IqG8|a(O&F23D00960h#R}Z HHT?hpriMol literal 5490 zcmV-&6^-g2iwFP!00000|Li?$bKADEe}%*QCC(^{UX~v-{b0*!qEjcw*Gk%R+IVgt z60%TJAVWZo6_5V=9RMW72S9=pzI z`WQHA{ET0IF)(sO%g@w@bm?Q`C5GQpFb`VB8{hxn(#g{ET%SUl^;omNf==p}yzn{< z?}N{H{TYAx^2KbKWVu)Xe`TJUALr1A2JoSQU23c#H2_{2mb+L$Oo=gbeS^*sF~p>| z&6eo_A1uU_Eqylm)72}_C5X>(@N;5uyl!S0Qd5|~h<&l>!^L)$#1{t_}|JRe$s zvM0RP`i326@pqWT#GO?lUTh)biSM#K3upI8V({ru zN!xN4uV(@Kz4CmQP5SF8W2@Iw$DO`j079Yv8hGfn=erN9|GR~_ee3?uWp8+FuGg(B z51%`(bw{Y{!?%!@s~RDK1Up`rLRln5JUF z=5vdwF}5M5Xoe(2K0>zPGa4iuV_*>I%wCB$Q-p8DI$;{hR%jb2g4Q<1l<~*R^%u}K zrYi#&zYEpSGJr91ZD<)UzRi&}HxRZFqjUC-#nPe3V!yuffOQA$=q-$k4h(>80~qgF zt2rR^c9QQN&DU(+z--lkj` zij<=h4UtmbFvxT-7A*)TB*)JtIT+PtW;S?OY+JB5#js)hAizJg zBTgb+)7AS8oSFsLBPg+vMIkb~;ZB}G8l zS{ANoYup282`gK-GVb`Wbh6wLJdDY#NDn&Bk4^jCo(Ao|9c@AJZ$}b3=U#c(f>YQv zUx{3Gn1q_^rb|GV^V%~b*qZd^zMce8iR&kCRm>-UJ(|D46<0vFU3;O*C zfxCX+CNnZe!`=~?ejlL0aCm6Xmf!vU*BKd17r-9c4}FAg?XPCboGdL1l1%{a4O=Ml z)cm~cbov%wHvSQs>zkXFi6{&>J~xc##U4F|Fhd3l*?HX(!;n0Wu!?>QRnmGb2@=Lc z1@L{a`mvd}*Vka)F-8;t2YrXz>aG-|dem16XssG7i*TdF_ECuq4mJ4T5)sPDpsZ6T z2co6a#(3sxtF#SlebvTZP+hSlM)MR3j-gwm-hwtytlTJ|3}21`|8jN@kOQU;G@p&C z6F6pm*?&EEIMm?pD+Db_7%KvV5x$K)@-5h12~(x}7Eu@=YRaWT5Czee1 zR9PV8jd$m&Q!;mYdi4C8*y7mmVU=KKCbn6ThvZ^#69!8Qzrp}nQsmVVPLfX69tGes zmehX9!{L#JJiLaMdk_6>js7^8dMy4p#9Gb$6snD_zmK;5kt&@eNeQ=z0ob1SV{q!C z{us!b$v=r~L;w3o|HrD_2TlqQWG6#|+#zsO)V*9HyZ!0Y_~O8L@scdqM3VNE@7^OD z`m7^&ONtF!2pl=JW8{>#R|Ud5r|@X-dPLqi692!F{1yJc8$)#g9>xM_ci1u)2*HbLZl~XCnZHAyS;IDG)nNVwV&-X>``R+Yd5b^0 z1XJjUS({Mc(>I=L%^BEkr`s~$U_1F>LR}x+!YdzH@Cy3Dk7g$ZedIb0pwMTS@E-&s zsmJT^`2^i!AeiaV{K%=Q*D@~v1?Fis=WM-hnPc+r*~~#0CP#V&q0c7S?Dj7Dzq#%o zZ@)i$|LxEJ-otnQq2Al!8Sy{;>)iU|;obSKlioY>#(n$o!GC{w|J#4)%xrP$V6VM) zU0UB#jq%i7Vw*|SD?sOwf(w*Z%lzmAOlHvE3_smM437^j^~+TtZnE>#G6i`IdTgTS z0AV&f2IFwR2vtkvim%#S(6CWphxh1ot&6*?az1*=E4Fthc=+_6-5iU>9q;i$>mLn zu}j#lL~p^EkLCi}h<}(O4E)s_Y`HeWkns$GLr(h{+y#9X&&{T95PMv=%&kUS*eUb6 ztk5b8VIw?rliEoVf{z&)x3| z(kqHRtSo<)JmU2{-S$C8m2Dilv*#vrE4#FXT9FOd{9mH z^3RsN+yo(-AVd>{Xo3(;5TXe}G(m_a2+;%~njl0IglK{gO%OsO2+_?w=(t=gR#g)S z!oJEJnZOm}b_ieQ1LMWZxMOw=**K7k^>XIJ7uFbMh>cNwEPpWW# z8V~g8JkZ^^icjDPUc-fZ4?kf7fdlQ4>#f4In%p6e`9h&*RLQMse3QmEX?&9($Tz9M zkN6}O_@DD53JiUj$dk3p3P4$_#!N09;8z-j*#LiDQypXK%(9?IHyB<+Kw_EAfwn9g zD~y23$xFOK>lEJ)>}vG#>Pn2U4IiK%vd)>~uobLwE>{6bdN?;fKVJcNpoc)*PZ+<3r^2i$nTjR)L# zz>No7#?2TEa@>r70ok(sf#{s;;qo4b1WMsC-j?C@q9g0uxF3?2*XN1IN5)3N@Ndri z?&8evhE~1a^E~U{NXEU8jE9PHeLN^Oaxr1+7D#|7v!_CYuo#}ah+k1%_F}j-6>HZT zz)tG+HATYHL_1W}pp&gp;IPz0vTa^~56mUN70oz|}ysGO)WKSq1tH!9EGW_7rCu z_%eu%HDvSK&m2Tb-Zv|Up}GM;qt>?^VC}WNm3Z%{)_+svx+!v98$@1 zC@cc0+F4=JGbg!F3--2IhPngTc~;lcsGuf`@$zt>$YNBG!j>jlKEQfequ5T{nlVzy zH*B#tY;maAbewvxjmB4U6mziyY@|CC6x?rdV(R#G2ScKO|h8 zITYwOtg$z&agw)bJ%O~2sIv@=iU~GB;$lwF3_h19e;Kwrr}-wp{G0>KjjXsFb`*&K z?}2{@=?Gv8@^eMeahX!0LIp)7DQrLi4cfHB#=+SD4?@4qKlh8&s_M1d%95Pnghv1v zdOH;t<&~{#nj8&uT%)RYJPtV?S5m!{9|&dr^xdVEch1E zGjdl&7KxtnE%KK{n8XX-vMmbx2WIjaxy#!)j{~pev{4G%ootL3j8{^)zNo#{0Tyy6 zsu~~*2B-Q%mn8n|J;IsN1nuV2snTKO+`4A9MGNqj4&VbjdUZHxuUg?YYsLM*u z!~|88Zu=n`4i)()fRBtY)`Ufwk zdj6;7e>PWks7U=0EP9CujmMHoZcZtHsB*v^;GuTB*qAp3iQV!cc*<`dWtRyzliD_ka{cpjma1ziU4)}kaxyp0x_?%CBFae zF0DC~Ty6Cp5&~{v_J@DdIrOg}?^E%jvTzDGPc!pvLcy&tzrs&{J(?fH;-p@GArWhv z;(FpD8s;LZZ}R%&*~N`6-CKdhrF)xwj|PbK&H84Y+L0+?hQ?^2TtTnInoF;qi6}t! zYqac=B3H4Mm)+J9tzn5_ZaPK9$I6uCiZ6Fr!#P1nlJN)%sD>CP8VSq<-E1F%sdwbsA&M7OG()nZ+mRunKwDyms9S28Ax&-J9n}`zl#(5Z3+n2b(KFlr zb%|}lfr<-H!&}!jPr@ycigsDFAQk(ItBgfJjuAj`Uu@fP>5?oMZm(yAF1&89&n-*} zY(PPNtBC{1Jh{ZuJ5`nTDxbA$NTAQJzT*pi4QG5Y15F~Y)Ld_6=r9W@TD&-i^ad^S z8hqm!O0iPUXZHX(VCukMT|YD6i^O%f+!R7C&bMk)F3XtHXUtx#y71SKc&;!itAgO} zP;}{^$W7ok=mu9Mj?S*LrS7U)*9m4SYhf!^MS25oR2@14DMSA3%aCVGoRVu2vs=pF7CS4+8XLvC zW7S4FInb=%__Ss5G=?jQIOz1z zi#>pq;{voBQ2QHrLIIa8Y=2)c)}s22Wr6{rJkV(yN)j!zH4;J9!XfD$xT;Ga z_p!Fg3*1Oa*=4BJ%UuDUZ4+x&bP-S%ZjRM2175+*PmN+HJA*P*Gc&GeqHn1tPI4DROA(VHmr zbp?Io+8k^3W=*24c{=Pw|CLN>-%3mWp!*a}9KDI7Kl?bkmPMJA0$P(V5i*bF%=H(5 zigV=f6-5h}V}|asQ-)Digy>xf7zKV=CqYY|D?=om!x?=bR2>`T|;XH~>YuAviVhA>kqan`C;`%dTj zrldhueOD$u)@u8FB~#a{?6IB`@PmGiTy96^1Vxs4w0CN?#w9$tH%0_8|igDT>(P3~tkffDsEXU<05 zwB$?Fx?Ynuj(a0vel`gsCC6N|Hr`4r()SF{QaoYqSd-zsx2ZM9in>BVv|b`YN62QJ zQZg?%+kcBo?X$pYztCQaZ*!Ub9;mL2n%S$w{-YDcdEX5|i%(VASA434dBvw4vTlTH z!?+FOK6A!B?kG;=TrV+23piO$$%^pxEycN+?m6dicyXIe%RDa5fGs+XlY(7}qh(`| zvvd|M0gV`R4h^z-Oy_{|JwJ%S)M0N&SZwnlOenw>Y=6deob5@CTYk1D+ekxfm?B4l zm~zf96DY|9M)r%fx{7>p;X{yXI4j9iv5synHkS-tK*iR9Ctb${%PYKb~~N8<2*P%JQ{XdW_VPMe;3}f2Pm7E otAwlDep%H?-jy>O;@)hsWca*y!Z+ytKL7y#|I28LQX2>X0PT*HO8@`> diff --git a/cmd/boost/retrieve_cmd.go b/cmd/boost/retrieve_cmd.go index 50c10dfe0..c002c5f7f 100644 --- a/cmd/boost/retrieve_cmd.go +++ b/cmd/boost/retrieve_cmd.go @@ -7,11 +7,6 @@ import ( "strings" "time" - flatfs "github.com/ipfs/go-ds-flatfs" - levelds "github.com/ipfs/go-ds-leveldb" - blockstore "github.com/ipfs/go-ipfs-blockstore" - "github.com/mitchellh/go-homedir" - "github.com/dustin/go-humanize" clinode "github.com/filecoin-project/boost/cli/node" "github.com/filecoin-project/boost/cmd" @@ -23,6 +18,9 @@ import ( lcli "github.com/filecoin-project/lotus/cli" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" + flatfs "github.com/ipfs/go-ds-flatfs" + levelds "github.com/ipfs/go-ds-leveldb" + blockstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" files "github.com/ipfs/go-ipfs-files" "github.com/ipfs/go-merkledag" @@ -35,6 +33,7 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/ipld/go-ipld-prime/traversal/selector/builder" textselector "github.com/ipld/go-ipld-selector-text-lite" + "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" "golang.org/x/term" "golang.org/x/xerrors" diff --git a/cmd/boostci/utils_cmd.go b/cmd/boostci/utils_cmd.go index 35f4abf66..3418a224b 100644 --- a/cmd/boostci/utils_cmd.go +++ b/cmd/boostci/utils_cmd.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "github.com/docker/go-units" "github.com/filecoin-project/boostd-data/shared/cliutil" "github.com/filecoin-project/go-paramfetch" diff --git a/cmd/boostd/index.go b/cmd/boostd/index.go index f4947d005..3008ea184 100644 --- a/cmd/boostd/index.go +++ b/cmd/boostd/index.go @@ -1,8 +1,10 @@ package main import ( + "fmt" bcli "github.com/filecoin-project/boost/cli" lcli "github.com/filecoin-project/lotus/cli" + "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" ) @@ -11,6 +13,7 @@ var indexProvCmd = &cli.Command{ Usage: "Manage the index provider on Boost", Subcommands: []*cli.Command{ indexProvAnnounceAllCmd, + indexProvListMultihashesCmd, }, } @@ -21,13 +24,51 @@ var indexProvAnnounceAllCmd = &cli.Command{ Action: func(cctx *cli.Context) error { ctx := lcli.ReqContext(cctx) - // announce markets and boost deals + // get boost api napi, closer, err := bcli.GetBoostAPI(cctx) if err != nil { return err } defer closer() + // announce markets and boost deals return napi.BoostIndexerAnnounceAllDeals(ctx) }, } + +var indexProvListMultihashesCmd = &cli.Command{ + Name: "list-multihashes", + Usage: "list-multihashes ", + UsageText: "List multihashes for a deal by proposal cid", + Action: func(cctx *cli.Context) error { + if cctx.NArg() != 1 { + return fmt.Errorf("must supply proposal cid") + } + + propCid, err := cid.Parse(cctx.Args().First()) + if err != nil { + return fmt.Errorf("parsing proposal cid %s: %w", cctx.Args().First(), err) + } + + ctx := lcli.ReqContext(cctx) + + // get boost api + napi, closer, err := bcli.GetBoostAPI(cctx) + if err != nil { + return err + } + defer closer() + + // get list of multihashes + mhs, err := napi.BoostIndexerListMultihashes(ctx, propCid) + if err != nil { + return err + } + + fmt.Printf("Found %d multihashes for deal with proposal cid %s:\n", len(mhs), propCid) + for _, mh := range mhs { + fmt.Println(" " + mh.String()) + } + return nil + }, +} diff --git a/cmd/boostd/main.go b/cmd/boostd/main.go index 0d9fd21e6..a5ea188ce 100644 --- a/cmd/boostd/main.go +++ b/cmd/boostd/main.go @@ -3,12 +3,11 @@ package main import ( "os" + "github.com/filecoin-project/boost/build" "github.com/filecoin-project/boost/cmd" + "github.com/filecoin-project/boostd-data/shared/cliutil" logging "github.com/ipfs/go-log/v2" "github.com/urfave/cli/v2" - - "github.com/filecoin-project/boost/build" - "github.com/filecoin-project/boostd-data/shared/cliutil" ) var log = logging.Logger("boostd") @@ -48,8 +47,8 @@ func main() { importDataCmd, logCmd, dagstoreCmd, - piecesCmd, netCmd, + pieceDirCmd, }, } app.Setup() @@ -66,6 +65,7 @@ func before(cctx *cli.Context) error { _ = logging.SetLogLevel("modules", "INFO") _ = logging.SetLogLevel("cfg", "INFO") _ = logging.SetLogLevel("boost-storage-deal", "INFO") + _ = logging.SetLogLevel("piecedir", "INFO") _ = logging.SetLogLevel("index-provider-wrapper", "INFO") if cliutil.IsVeryVerbose { @@ -78,6 +78,8 @@ func before(cctx *cli.Context) error { _ = logging.SetLogLevel("boost-migrator", "DEBUG") _ = logging.SetLogLevel("dagstore", "DEBUG") _ = logging.SetLogLevel("migrator", "DEBUG") + _ = logging.SetLogLevel("piecedir", "DEBUG") + _ = logging.SetLogLevel("fxlog", "DEBUG") } return nil diff --git a/cmd/boostd/piecedir.go b/cmd/boostd/piecedir.go new file mode 100644 index 000000000..da8f04628 --- /dev/null +++ b/cmd/boostd/piecedir.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + _ "net/http/pprof" + "time" + + bcli "github.com/filecoin-project/boost/cli" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/ipfs/go-cid" + "github.com/urfave/cli/v2" +) + +var pieceDirCmd = &cli.Command{ + Name: "lid", + Usage: "Manage Local Index Directory", + Subcommands: []*cli.Command{ + pdIndexGenerate, + pdIndexMarkErroredCmd, + }, +} + +var pdIndexGenerate = &cli.Command{ + Name: "gen-index", + Usage: "Generate index for a given piece from the piece data stored in a sector", + ArgsUsage: "", + Action: func(cctx *cli.Context) error { + ctx := lcli.ReqContext(cctx) + + if cctx.Args().Len() != 1 { + return fmt.Errorf("must specify piece CID") + } + + // parse piececid + piececid, err := cid.Decode(cctx.Args().Get(0)) + if err != nil { + return fmt.Errorf("parsing piece CID: %w", err) + } + fmt.Println("Generating index for piece", piececid) + + boostApi, ncloser, err := bcli.GetBoostAPI(cctx) + if err != nil { + return fmt.Errorf("getting boost api: %w", err) + } + defer ncloser() + + addStart := time.Now() + + err = boostApi.PdBuildIndexForPieceCid(ctx, piececid) + if err != nil { + return err + } + + fmt.Println("Generated index in", time.Since(addStart).String()) + + return nil + }, +} + +var pdIndexMarkErroredCmd = &cli.Command{ + Name: "mark-index", + Usage: "Mark an index errored for a given piece in the local index directory", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "piece-cid", + Usage: "piece-cid of the index that will be marked as errored", + Required: true, + }, + &cli.StringFlag{ + Name: "error", + Usage: "error message", + Required: true, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := lcli.ReqContext(cctx) + + // parse piececid + piececid, err := cid.Decode(cctx.String("piece-cid")) + if err != nil { + return err + } + + boostApi, ncloser, err := bcli.GetBoostAPI(cctx) + if err != nil { + return fmt.Errorf("getting boost api: %w", err) + } + defer ncloser() + + errMsg := cctx.String("error") + err = boostApi.PdMarkIndexErrored(ctx, piececid, errMsg) + if err != nil { + return err + } + + fmt.Printf("Marked %s as errored with \"%s\"\n", piececid, errMsg) + + return nil + }, +} diff --git a/cmd/boostd/pieces.go b/cmd/boostd/pieces.go deleted file mode 100644 index 0bf4951b1..000000000 --- a/cmd/boostd/pieces.go +++ /dev/null @@ -1,290 +0,0 @@ -package main - -import ( - "fmt" - "os" - "text/tabwriter" - - bcli "github.com/filecoin-project/boost/cli" - "github.com/filecoin-project/boost/cmd" - lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/lib/tablewriter" - "github.com/ipfs/go-cid" - "github.com/urfave/cli/v2" -) - -var piecesCmd = &cli.Command{ - Name: "pieces", - Usage: "Interact with the Piece Store", - Description: "The piecestore is a database that tracks and manages data that is made available to the retrieval market", - Subcommands: []*cli.Command{ - piecesListPiecesCmd, - piecesListCidInfosCmd, - piecesInfoCmd, - piecesCidInfoCmd, - }, -} - -var piecesListPiecesCmd = &cli.Command{ - Name: "list-pieces", - Usage: "List registered pieces", - Action: func(cctx *cli.Context) error { - nodeApi, closer, err := bcli.GetBoostAPI(cctx) - if err != nil { - return err - } - defer closer() - ctx := lcli.ReqContext(cctx) - - pieceCids, err := nodeApi.PiecesListPieces(ctx) - if err != nil { - return err - } - - if cctx.Bool("json") { - - pieceCidsJson := map[string]interface{}{ - "pieceCids": pieceCids, - } - return cmd.PrintJson(pieceCidsJson) - } - - for _, pc := range pieceCids { - fmt.Println(pc) - } - - return nil - }, -} - -var piecesListCidInfosCmd = &cli.Command{ - Name: "list-cids", - Usage: "List registered payload CIDs", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "verbose", - Aliases: []string{"v"}, - }, - }, - Action: func(cctx *cli.Context) error { - nodeApi, closer, err := bcli.GetBoostAPI(cctx) - if err != nil { - return err - } - defer closer() - ctx := lcli.ReqContext(cctx) - - cids, err := nodeApi.PiecesListCidInfos(ctx) - if err != nil { - return err - } - - if cctx.Bool("json") && !cctx.Bool("verbose") { - dataCidsJson := map[string]interface{}{ - "dataCids": cids, - } - - return cmd.PrintJson(dataCidsJson) - } - - w := tablewriter.New(tablewriter.Col("CID"), - tablewriter.Col("Piece"), - tablewriter.Col("BlockOffset"), - tablewriter.Col("BlockLen"), - tablewriter.Col("Deal"), - tablewriter.Col("Sector"), - tablewriter.Col("DealOffset"), - tablewriter.Col("DealLen"), - ) - - type dataCids map[string]interface{} - var out []dataCids - - for _, c := range cids { - if !cctx.Bool("verbose") { - fmt.Println(c) - continue - } - - type pbl map[string]interface{} - var pbls []pbl - - ci, err := nodeApi.PiecesGetCIDInfo(ctx, c) - if err != nil { - fmt.Printf("Error getting CID info: %s\n", err) - continue - } - - for _, location := range ci.PieceBlockLocations { - pi, err := nodeApi.PiecesGetPieceInfo(ctx, location.PieceCID) - if err != nil { - fmt.Printf("Error getting piece info: %s\n", err) - continue - } - - for _, deal := range pi.Deals { - w.Write(map[string]interface{}{ - "CID": c, - "Piece": location.PieceCID, - "BlockOffset": location.RelOffset, - "BlockLen": location.BlockSize, - "Deal": deal.DealID, - "Sector": deal.SectorID, - "DealOffset": deal.Offset, - "DealLen": deal.Length, - }) - - deal := map[string]interface{}{ - "ID": deal.DealID, - "Sector": deal.SectorID, - "Offset": deal.Offset, - "Length": deal.Length, - } - tpbl := pbl{ - "PieceCid": pi.PieceCID, - "BlockOffset": location.RelOffset, - "BlockLength": location.BlockSize, - "Deal": deal, - } - - pbls = append(pbls, tpbl) - } - } - - tdataCids := dataCids{ - "DataCid": c, - "PieceBlockLocation": pbls, - } - - out = append(out, tdataCids) - } - - if cctx.Bool("json") && cctx.Bool("verbose") { - return cmd.PrintJson(out) - } - - if cctx.Bool("verbose") { - return w.Flush(os.Stdout) - } - - return nil - }, -} - -var piecesInfoCmd = &cli.Command{ - Name: "piece-info", - Usage: "Get registered information for a given piece CID", - Action: func(cctx *cli.Context) error { - if !cctx.Args().Present() { - return lcli.ShowHelp(cctx, fmt.Errorf("must specify piece cid")) - } - - nodeApi, closer, err := bcli.GetBoostAPI(cctx) - if err != nil { - return err - } - defer closer() - ctx := lcli.ReqContext(cctx) - - c, err := cid.Decode(cctx.Args().First()) - if err != nil { - return err - } - - pi, err := nodeApi.PiecesGetPieceInfo(ctx, c) - if err != nil { - return err - } - - if cctx.Bool("json") { - - type deal map[string]interface{} - var deals []deal - for _, d := range pi.Deals { - dl := deal{ - "ID": d.DealID, - "SectorID": d.SectorID, - "Offset": d.Offset, - "Length": d.Length, - } - deals = append(deals, dl) - } - - pieceinfo := map[string]interface{}{ - "PieceCid": pi.PieceCID, - "Deals": deals, - } - - return cmd.PrintJson(pieceinfo) - } - - fmt.Println("Piece: ", pi.PieceCID) - w := tabwriter.NewWriter(os.Stdout, 4, 4, 2, ' ', 0) - fmt.Fprintln(w, "Deals:\nDealID\tSectorID\tLength\tOffset") - for _, d := range pi.Deals { - fmt.Fprintf(w, "%d\t%d\t%d\t%d\n", d.DealID, d.SectorID, d.Length, d.Offset) - } - return w.Flush() - }, -} - -var piecesCidInfoCmd = &cli.Command{ - Name: "cid-info", - Usage: "Get registered information for a given payload CID", - Action: func(cctx *cli.Context) error { - if !cctx.Args().Present() { - return lcli.ShowHelp(cctx, fmt.Errorf("must specify payload cid")) - } - - nodeApi, closer, err := bcli.GetBoostAPI(cctx) - if err != nil { - return err - } - defer closer() - ctx := lcli.ReqContext(cctx) - - c, err := cid.Decode(cctx.Args().First()) - if err != nil { - return err - } - - ci, err := nodeApi.PiecesGetCIDInfo(ctx, c) - if err != nil { - return err - } - - if cctx.Bool("json") { - - type pbl map[string]interface{} - type bl map[string]interface{} - var pbls []pbl - for _, d := range ci.PieceBlockLocations { - tbp := bl{ - "RelOffset": d.RelOffset, - "BlockSize": d.BlockSize, - } - tpbl := pbl{ - "PieceCid": d.PieceCID, - "BlockLocation": tbp, - } - pbls = append(pbls, tpbl) - } - - pieceinfo := map[string]interface{}{ - "DataCid": ci.CID, - "PieceBlockLocation": pbls, - } - - return cmd.PrintJson(pieceinfo) - } - - fmt.Println("Info for: ", ci.CID) - - w := tabwriter.NewWriter(os.Stdout, 4, 4, 2, ' ', 0) - fmt.Fprintf(w, "PieceCid\tOffset\tSize\n") - for _, loc := range ci.PieceBlockLocations { - fmt.Fprintf(w, "%s\t%d\t%d\n", loc.PieceCID, loc.RelOffset, loc.BlockSize) - } - return w.Flush() - }, -} diff --git a/cmd/booster-bitswap/init.go b/cmd/booster-bitswap/init.go index c6d74ca87..d79a5fb5e 100644 --- a/cmd/booster-bitswap/init.go +++ b/cmd/booster-bitswap/init.go @@ -25,10 +25,6 @@ func configureRepo(cfgDir string, createIfNotExist bool) (peer.ID, crypto.PrivKe return "", nil, fmt.Errorf("%s is a required flag", FlagRepo.Name) } - if err := os.MkdirAll(cfgDir, 0744); err != nil { - return "", nil, err - } - peerkey, err := loadPeerKey(cfgDir, createIfNotExist) if err != nil { return "", nil, err @@ -72,7 +68,9 @@ func loadPeerKey(cfgDir string, createIfNotExists bool) (crypto.PrivKey, error) return nil, err } if !createIfNotExists { - return nil, fmt.Errorf("booster-bitswap has not been initialized. Run the booster-bitswap init command") + msg := keyPath + " not found: " + msg += "booster-bitswap has not been initialized. Run the booster-bitswap init command" + return nil, fmt.Errorf(msg) } log.Infof("Generating new peer key...") @@ -86,6 +84,9 @@ func loadPeerKey(cfgDir string, createIfNotExists bool) (crypto.PrivKey, error) return nil, err } + if err := os.MkdirAll(cfgDir, 0744); err != nil { + return nil, err + } if err := os.WriteFile(keyPath, data, 0600); err != nil { return nil, err } diff --git a/cmd/booster-bitswap/run.go b/cmd/booster-bitswap/run.go index 9fbb9379b..3fc52b833 100644 --- a/cmd/booster-bitswap/run.go +++ b/cmd/booster-bitswap/run.go @@ -1,20 +1,16 @@ package main import ( - "context" "fmt" "net/http" _ "net/http/pprof" - "strings" - "github.com/filecoin-project/boost/api" - bclient "github.com/filecoin-project/boost/api/client" - cliutil "github.com/filecoin-project/boost/cli/util" "github.com/filecoin-project/boost/cmd/booster-bitswap/filters" "github.com/filecoin-project/boost/cmd/booster-bitswap/remoteblockstore" + "github.com/filecoin-project/boost/cmd/lib" "github.com/filecoin-project/boost/metrics" + "github.com/filecoin-project/boost/piecedirectory" "github.com/filecoin-project/boostd-data/shared/tracing" - "github.com/filecoin-project/go-jsonrpc" lcli "github.com/filecoin-project/lotus/cli" "github.com/libp2p/go-libp2p/core/peer" "github.com/mitchellh/go-homedir" @@ -46,10 +42,25 @@ var runCmd = &cli.Command{ Value: 9696, }, &cli.StringFlag{ - Name: "api-boost", - Usage: "the endpoint for the boost API", + Name: "api-fullnode", + Usage: "the endpoint for the full node API", Required: true, }, + &cli.StringFlag{ + Name: "api-storage", + Usage: "the endpoint for the storage node API", + Required: true, + }, + &cli.StringFlag{ + Name: "api-lid", + Usage: "the endpoint for the local index directory API, eg 'http://localhost:8042'", + Required: true, + }, + &cli.IntFlag{ + Name: "add-index-throttle", + Usage: "the maximum number of add index operations that can run in parallel", + Value: 4, + }, &cli.StringFlag{ Name: "proxy", Usage: "the multiaddr of the libp2p proxy that this node connects through", @@ -129,20 +140,35 @@ var runCmd = &cli.Command{ }() } - // Connect to the Boost API - boostAPIInfo := cctx.String("api-boost") - bapi, bcloser, err := getBoostAPI(ctx, boostAPIInfo) + // Connect to the full node API + fnApiInfo := cctx.String("api-fullnode") + fullnodeApi, ncloser, err := lib.GetFullNodeApi(ctx, fnApiInfo, log) + if err != nil { + return fmt.Errorf("getting full node API: %w", err) + } + defer ncloser() + + // Connect to the storage API and create a sector accessor + storageApiInfo := cctx.String("api-storage") + sa, storageCloser, err := lib.CreateSectorAccessor(ctx, storageApiInfo, fullnodeApi, log) if err != nil { - return fmt.Errorf("getting boost API: %w", err) + return err } - defer bcloser() + defer storageCloser() - remoteStore := remoteblockstore.NewRemoteBlockstore(bapi) - // Create the server API + // Connect to the local index directory service + pdClient := piecedirectory.NewStore() + defer pdClient.Close(ctx) + err = pdClient.Dial(ctx, cctx.String("api-lid")) + if err != nil { + return fmt.Errorf("connecting to local index directory service: %w", err) + } + + // Create the bitswap host port := cctx.Int("port") repoDir, err := homedir.Expand(cctx.String(FlagRepo.Name)) if err != nil { - return fmt.Errorf("expanding repo file path: %w", err) + return fmt.Errorf("expanding repo file path %s: %w", cctx.String(FlagRepo.Name), err) } host, err := setupHost(repoDir, port) if err != nil { @@ -155,6 +181,9 @@ var runCmd = &cli.Command{ if err != nil { return fmt.Errorf("starting block filter: %w", err) } + pr := &piecedirectory.SectorAccessorAsPieceReader{SectorAccessor: sa} + piecedirectory := piecedirectory.NewPieceDirectory(pdClient, pr, cctx.Int("add-index-throttle")) + remoteStore := remoteblockstore.NewRemoteBlockstore(piecedirectory) server := NewBitswapServer(remoteStore, host, multiFilter) var proxyAddrInfo *peer.AddrInfo @@ -166,6 +195,9 @@ var runCmd = &cli.Command{ } } + // Start the local index directory + piecedirectory.Start(ctx) + // Start the bitswap server log.Infof("Starting booster-bitswap node on port %d", port) err = server.Start(ctx, proxyAddrInfo, &BitswapServerOptions{ @@ -206,20 +238,3 @@ var runCmd = &cli.Command{ return nil }, } - -func getBoostAPI(ctx context.Context, ai string) (api.Boost, jsonrpc.ClientCloser, error) { - ai = strings.TrimPrefix(strings.TrimSpace(ai), "BOOST_API_INFO=") - info := cliutil.ParseApiInfo(ai) - addr, err := info.DialArgs("v0") - if err != nil { - return nil, nil, fmt.Errorf("could not get DialArgs: %w", err) - } - - log.Infof("Using boost API at %s", addr) - api, closer, err := bclient.NewBoostRPCV0(ctx, addr, info.AuthHeader()) - if err != nil { - return nil, nil, fmt.Errorf("creating full node service API: %w", err) - } - - return api, closer, nil -} diff --git a/cmd/booster-http/http_test.go b/cmd/booster-http/http_test.go index 54e7b7feb..1183abf62 100644 --- a/cmd/booster-http/http_test.go +++ b/cmd/booster-http/http_test.go @@ -9,10 +9,10 @@ import ( "os" "testing" + "github.com/filecoin-project/boostd-data/model" + mocks_booster_http "github.com/filecoin-project/boost/cmd/booster-http/mocks" - "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/golang/mock/gomock" - "github.com/ipfs/go-cid" "github.com/stretchr/testify/require" ) @@ -22,7 +22,7 @@ func TestNewHttpServer(t *testing.T) { // Create a new mock Http server ctrl := gomock.NewController(t) - httpServer := NewHttpServer("", 7777, false, mocks_booster_http.NewMockHttpServerApi(ctrl)) + httpServer := NewHttpServer("", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl)) httpServer.Start(context.Background()) // Check that server is up @@ -40,7 +40,7 @@ func TestHttpGzipResponse(t *testing.T) { // Create a new mock Http server with custom functions ctrl := gomock.NewController(t) mockHttpServer := mocks_booster_http.NewMockHttpServerApi(ctrl) - httpServer := NewHttpServer("", 7777, false, mockHttpServer) + httpServer := NewHttpServer("", 7778, mockHttpServer) httpServer.Start(context.Background()) // Create mock unsealed file for piece/car @@ -51,31 +51,22 @@ func TestHttpGzipResponse(t *testing.T) { require.NoError(t, err) defer f.Close() - //Create CID - cid, err := cid.Parse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi") - require.NoError(t, err) - // Crate pieceInfo - deal := piecestore.DealInfo{ - DealID: 1234567, - SectorID: 0, - Offset: 1233, - Length: 123, - } - var deals []piecestore.DealInfo - - pieceInfo := piecestore.PieceInfo{ - PieceCID: cid, - Deals: append(deals, deal), + deal := model.DealInfo{ + ChainDealID: 1234567, + SectorID: 0, + PieceOffset: 1233, + PieceLength: 123, } + deals := []model.DealInfo{deal} mockHttpServer.EXPECT().UnsealSectorAt(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(f, nil) mockHttpServer.EXPECT().IsUnsealed(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(true, nil) - mockHttpServer.EXPECT().GetPieceInfo(gomock.Any()).AnyTimes().Return(&pieceInfo, nil) + mockHttpServer.EXPECT().GetPieceDeals(gomock.Any(), gomock.Any()).AnyTimes().Return(deals, nil) //Create a client and make request with Encoding header client := new(http.Client) - request, err := http.NewRequest("GET", "http://localhost:7777/piece/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", nil) + request, err := http.NewRequest("GET", "http://localhost:7778/piece/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi", nil) require.NoError(t, err) request.Header.Add("Accept-Encoding", "gzip") @@ -105,10 +96,10 @@ func TestHttpInfo(t *testing.T) { // Create a new mock Http server ctrl := gomock.NewController(t) - httpServer := NewHttpServer("", 7777, false, mocks_booster_http.NewMockHttpServerApi(ctrl)) + httpServer := NewHttpServer("", 7780, mocks_booster_http.NewMockHttpServerApi(ctrl)) httpServer.Start(context.Background()) - response, err := http.Get("http://localhost:7777/info") + response, err := http.Get("http://localhost:7780/info") require.NoError(t, err) defer response.Body.Close() @@ -118,5 +109,4 @@ func TestHttpInfo(t *testing.T) { // Stop the server err = httpServer.Stop() require.NoError(t, err) - } diff --git a/cmd/booster-http/mocks/mock_booster_http.go b/cmd/booster-http/mocks/mock_booster_http.go index 7b3e914b1..9a143b56d 100644 --- a/cmd/booster-http/mocks/mock_booster_http.go +++ b/cmd/booster-http/mocks/mock_booster_http.go @@ -8,8 +8,8 @@ import ( context "context" reflect "reflect" + model "github.com/filecoin-project/boostd-data/model" mount "github.com/filecoin-project/dagstore/mount" - piecestore "github.com/filecoin-project/go-fil-markets/piecestore" abi "github.com/filecoin-project/go-state-types/abi" gomock "github.com/golang/mock/gomock" cid "github.com/ipfs/go-cid" @@ -38,19 +38,34 @@ func (m *MockHttpServerApi) EXPECT() *MockHttpServerApiMockRecorder { return m.recorder } -// GetPieceInfo mocks base method. -func (m *MockHttpServerApi) GetPieceInfo(pieceCID cid.Cid) (*piecestore.PieceInfo, error) { +// GetBlockByCid mocks base method. +func (m *MockHttpServerApi) GetBlockByCid(ctx context.Context, blockCid cid.Cid) ([]byte, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetPieceInfo", pieceCID) - ret0, _ := ret[0].(*piecestore.PieceInfo) + ret := m.ctrl.Call(m, "GetBlockByCid", ctx, blockCid) + ret0, _ := ret[0].([]byte) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetPieceInfo indicates an expected call of GetPieceInfo. -func (mr *MockHttpServerApiMockRecorder) GetPieceInfo(pieceCID interface{}) *gomock.Call { +// GetBlockByCid indicates an expected call of GetBlockByCid. +func (mr *MockHttpServerApiMockRecorder) GetBlockByCid(ctx, blockCid interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPieceInfo", reflect.TypeOf((*MockHttpServerApi)(nil).GetPieceInfo), pieceCID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBlockByCid", reflect.TypeOf((*MockHttpServerApi)(nil).GetBlockByCid), ctx, blockCid) +} + +// GetPieceDeals mocks base method. +func (m *MockHttpServerApi) GetPieceDeals(ctx context.Context, pieceCID cid.Cid) ([]model.DealInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPieceDeals", ctx, pieceCID) + ret0, _ := ret[0].([]model.DealInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPieceDeals indicates an expected call of GetPieceDeals. +func (mr *MockHttpServerApiMockRecorder) GetPieceDeals(ctx, pieceCID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPieceDeals", reflect.TypeOf((*MockHttpServerApi)(nil).GetPieceDeals), ctx, pieceCID) } // IsUnsealed mocks base method. diff --git a/cmd/booster-http/run.go b/cmd/booster-http/run.go index 0811a4343..2dd60b688 100644 --- a/cmd/booster-http/run.go +++ b/cmd/booster-http/run.go @@ -5,16 +5,12 @@ import ( "fmt" "net/http" _ "net/http/pprof" - "strings" - "github.com/filecoin-project/boost/api" - bclient "github.com/filecoin-project/boost/api/client" - cliutil "github.com/filecoin-project/boost/cli/util" "github.com/filecoin-project/boost/cmd/lib" + "github.com/filecoin-project/boost/piecedirectory" + "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/filecoin-project/dagstore/mount" - "github.com/filecoin-project/go-fil-markets/piecestore" - "github.com/filecoin-project/go-jsonrpc" "github.com/filecoin-project/go-state-types/abi" lcli "github.com/filecoin-project/lotus/cli" "github.com/filecoin-project/lotus/markets/dagstore" @@ -41,16 +37,16 @@ var runCmd = &cli.Command{ Usage: "the port the web server listens on", Value: 7777, }, - &cli.BoolFlag{ - Name: "allow-indexing", - Usage: "allow booster-http to build an index for a CAR file on the fly if necessary (requires doing an extra pass over the CAR file)", - Value: false, - }, &cli.StringFlag{ - Name: "api-boost", - Usage: "the endpoint for the boost API", + Name: "api-lid", + Usage: "the endpoint for the local index directory API, eg 'http://localhost:8042'", Required: true, }, + &cli.IntFlag{ + Name: "add-index-throttle", + Usage: "the maximum number of add index operations that can run in parallel", + Value: 4, + }, &cli.StringFlag{ Name: "api-fullnode", Usage: "the endpoint for the full node API", @@ -82,14 +78,14 @@ var runCmd = &cli.Command{ }() } - // Connect to the Boost API + // Connect to the local index directory service ctx := lcli.ReqContext(cctx) - boostApiInfo := cctx.String("api-boost") - bapi, bcloser, err := getBoostApi(ctx, boostApiInfo) + pdClient := piecedirectory.NewStore() + defer pdClient.Close(ctx) + err := pdClient.Dial(ctx, cctx.String("api-lid")) if err != nil { - return fmt.Errorf("getting boost API: %w", err) + return fmt.Errorf("connecting to local index directory service: %w", err) } - defer bcloser() // Connect to the full node API fnApiInfo := cctx.String("api-fullnode") @@ -99,7 +95,7 @@ var runCmd = &cli.Command{ } defer ncloser() - // Connect to the storage API + // Connect to the storage API and create a sector accessor storageApiInfo := cctx.String("api-storage") if err != nil { return fmt.Errorf("parsing storage API endpoint: %w", err) @@ -123,26 +119,22 @@ var runCmd = &cli.Command{ } defer storageCloser() - allowIndexing := cctx.Bool("allow-indexing") // Create the server API - sapi := serverApi{ctx: ctx, bapi: bapi, sa: sa} + pr := &piecedirectory.SectorAccessorAsPieceReader{SectorAccessor: sa} + piecedirectory := piecedirectory.NewPieceDirectory(pdClient, pr, cctx.Int("add-index-throttle")) + sapi := serverApi{ctx: ctx, piecedirectory: piecedirectory, sa: sa} server := NewHttpServer( cctx.String("base-path"), cctx.Int("port"), - allowIndexing, sapi, ) + // Start the local index directory + piecedirectory.Start(ctx) + // Start the server log.Infof("Starting booster-http node on port %d with base path '%s'", cctx.Int("port"), cctx.String("base-path")) - var indexingStr string - if allowIndexing { - indexingStr = "Enabled" - } else { - indexingStr = "Disabled" - } - log.Info("On-the-fly indexing of CAR files is " + indexingStr) server.Start(ctx) // Monitor for shutdown. @@ -171,15 +163,15 @@ var runCmd = &cli.Command{ } type serverApi struct { - ctx context.Context - bapi api.Boost - sa dagstore.SectorAccessor + ctx context.Context + piecedirectory *piecedirectory.PieceDirectory + sa dagstore.SectorAccessor } var _ HttpServerApi = (*serverApi)(nil) -func (s serverApi) GetPieceInfo(pieceCID cid.Cid) (*piecestore.PieceInfo, error) { - return s.bapi.PiecesGetPieceInfo(s.ctx, pieceCID) +func (s serverApi) GetPieceDeals(ctx context.Context, pieceCID cid.Cid) ([]model.DealInfo, error) { + return s.piecedirectory.GetPieceDeals(ctx, pieceCID) } func (s serverApi) IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) { @@ -189,20 +181,3 @@ func (s serverApi) IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, of func (s serverApi) UnsealSectorAt(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) { return s.sa.UnsealSectorAt(ctx, sectorID, offset, length) } - -func getBoostApi(ctx context.Context, ai string) (api.Boost, jsonrpc.ClientCloser, error) { - ai = strings.TrimPrefix(strings.TrimSpace(ai), "BOOST_API_INFO=") - info := cliutil.ParseApiInfo(ai) - addr, err := info.DialArgs("v0") - if err != nil { - return nil, nil, fmt.Errorf("could not get DialArgs: %w", err) - } - - log.Infof("Using boost API at %s", addr) - api, closer, err := bclient.NewBoostRPCV0(ctx, addr, info.AuthHeader()) - if err != nil { - return nil, nil, fmt.Errorf("creating full node service API: %w", err) - } - - return api, closer, nil -} diff --git a/cmd/booster-http/server.go b/cmd/booster-http/server.go index 1af69cb18..f07635469 100644 --- a/cmd/booster-http/server.go +++ b/cmd/booster-http/server.go @@ -14,9 +14,9 @@ import ( "github.com/NYTimes/gziphandler" "github.com/fatih/color" "github.com/filecoin-project/boost/metrics" + "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/filecoin-project/dagstore/mount" - "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-state-types/abi" "github.com/hashicorp/go-multierror" @@ -39,10 +39,9 @@ type apiVersion struct { } type HttpServer struct { - path string - port int - allowIndexing bool - api HttpServerApi + path string + port int + api HttpServerApi ctx context.Context cancel context.CancelFunc @@ -50,13 +49,13 @@ type HttpServer struct { } type HttpServerApi interface { - GetPieceInfo(pieceCID cid.Cid) (*piecestore.PieceInfo, error) + GetPieceDeals(ctx context.Context, pieceCID cid.Cid) ([]model.DealInfo, error) IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) UnsealSectorAt(ctx context.Context, sectorID abi.SectorNumber, pieceOffset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) } -func NewHttpServer(path string, port int, allowIndexing bool, api HttpServerApi) *HttpServer { - return &HttpServer{path: path, port: port, allowIndexing: allowIndexing, api: api} +func NewHttpServer(path string, port int, api HttpServerApi) *HttpServer { + return &HttpServer{path: path, port: port, api: api} } func (s *HttpServer) pieceBasePath() string { @@ -107,8 +106,9 @@ const idxPage = ` Download a raw piece by its piece CID - /piece/ + /piece/ + @@ -172,16 +172,16 @@ func (s *HttpServer) handleByPieceCid(w http.ResponseWriter, r *http.Request) { etag := pieceCid.String() w.Header().Set("Etag", etag) - serveContent(w, r, content) + serveContent(w, r, content, "application/piece") stats.Record(ctx, metrics.HttpPieceByCid200ResponseCount.M(1)) stats.Record(ctx, metrics.HttpPieceByCidRequestDuration.M(float64(time.Since(startTime).Milliseconds()))) } -func serveContent(w http.ResponseWriter, r *http.Request, content io.ReadSeeker) { +func serveContent(w http.ResponseWriter, r *http.Request, content io.ReadSeeker, contentType string) { // Set the Content-Type header explicitly so that http.ServeContent doesn't // try to do it implicitly - w.Header().Set("Content-Type", "application/piece") + w.Header().Set("Content-Type", contentType) var writer http.ResponseWriter @@ -260,19 +260,19 @@ func writeError(w http.ResponseWriter, r *http.Request, status int, msg string) func (s *HttpServer) getPieceContent(ctx context.Context, pieceCid cid.Cid) (io.ReadSeeker, error) { // Get the deals for the piece - pieceInfo, err := s.api.GetPieceInfo(pieceCid) + pieceDeals, err := s.api.GetPieceDeals(ctx, pieceCid) if err != nil { return nil, fmt.Errorf("getting sector info for piece %s: %w", pieceCid, err) } // Get the first unsealed deal - di, err := s.unsealedDeal(ctx, *pieceInfo) + di, err := s.unsealedDeal(ctx, pieceCid, pieceDeals) if err != nil { return nil, fmt.Errorf("getting unsealed CAR file: %w", err) } // Get the raw piece data from the sector - pieceReader, err := s.api.UnsealSectorAt(ctx, di.SectorID, di.Offset.Unpadded(), di.Length.Unpadded()) + pieceReader, err := s.api.UnsealSectorAt(ctx, di.SectorID, di.PieceOffset.Unpadded(), di.PieceLength.Unpadded()) if err != nil { return nil, fmt.Errorf("getting raw data from sector %d: %w", di.SectorID, err) } @@ -280,17 +280,17 @@ func (s *HttpServer) getPieceContent(ctx context.Context, pieceCid cid.Cid) (io. return pieceReader, nil } -func (s *HttpServer) unsealedDeal(ctx context.Context, pieceInfo piecestore.PieceInfo) (*piecestore.DealInfo, error) { +func (s *HttpServer) unsealedDeal(ctx context.Context, pieceCid cid.Cid, pieceDeals []model.DealInfo) (*model.DealInfo, error) { // There should always been deals in the PieceInfo, but check just in case - if len(pieceInfo.Deals) == 0 { - return nil, fmt.Errorf("there are no deals containing piece %s: %w", pieceInfo.PieceCID, ErrNotFound) + if len(pieceDeals) == 0 { + return nil, fmt.Errorf("there are no deals containing piece %s: %w", pieceCid, ErrNotFound) } // The same piece can be in many deals. Find the first unsealed deal. sealedCount := 0 var allErr error - for _, di := range pieceInfo.Deals { - isUnsealed, err := s.api.IsUnsealed(ctx, di.SectorID, di.Offset.Unpadded(), di.Length.Unpadded()) + for _, di := range pieceDeals { + isUnsealed, err := s.api.IsUnsealed(ctx, di.SectorID, di.PieceOffset.Unpadded(), di.PieceLength.Unpadded()) if err != nil { allErr = multierror.Append(allErr, err) continue @@ -304,29 +304,29 @@ func (s *HttpServer) unsealedDeal(ctx context.Context, pieceInfo piecestore.Piec // It wasn't possible to find a deal with the piece cid that is unsealed. // Try to return an error message with as much useful information as possible - dealSectors := make([]string, 0, len(pieceInfo.Deals)) - for _, di := range pieceInfo.Deals { - dealSectors = append(dealSectors, fmt.Sprintf("Deal %d: Sector %d", di.DealID, di.SectorID)) + dealSectors := make([]string, 0, len(pieceDeals)) + for _, di := range pieceDeals { + dealSectors = append(dealSectors, fmt.Sprintf("Deal %d: Sector %d", di.ChainDealID, di.SectorID)) } if allErr == nil { dealSectorsErr := fmt.Errorf("%s: %w", strings.Join(dealSectors, ", "), ErrNotFound) return nil, fmt.Errorf("checked unsealed status of %d deals containing piece %s: none are unsealed: %w", - len(pieceInfo.Deals), pieceInfo.PieceCID, dealSectorsErr) + len(pieceDeals), pieceCid, dealSectorsErr) } - if len(pieceInfo.Deals) == 1 { + if len(pieceDeals) == 1 { return nil, fmt.Errorf("checking unsealed status of deal %d (sector %d) containing piece %s: %w", - pieceInfo.Deals[0].DealID, pieceInfo.Deals[0].SectorID, pieceInfo.PieceCID, allErr) + pieceDeals[0].ChainDealID, pieceDeals[0].SectorID, pieceCid, allErr) } if sealedCount == 0 { return nil, fmt.Errorf("checking unsealed status of %d deals containing piece %s: %s: %w", - len(pieceInfo.Deals), pieceInfo.PieceCID, dealSectors, allErr) + len(pieceDeals), pieceCid, dealSectors, allErr) } return nil, fmt.Errorf("checking unsealed status of %d deals containing piece %s - %d are sealed, %d had errors: %s: %w", - len(pieceInfo.Deals), pieceInfo.PieceCID, sealedCount, len(pieceInfo.Deals)-sealedCount, dealSectors, allErr) + len(pieceDeals), pieceCid, sealedCount, len(pieceDeals)-sealedCount, dealSectors, allErr) } // writeErrorWatcher calls onError if there is an error writing to the writer diff --git a/cmd/migrate-lid/Makefile b/cmd/migrate-lid/Makefile new file mode 100644 index 000000000..f263ebf11 --- /dev/null +++ b/cmd/migrate-lid/Makefile @@ -0,0 +1,17 @@ +SHELL=/usr/bin/env bash + +unexport GOFLAGS + +GOCC?=go + +ldflags=-X=github.com/filecoin-project/boost/build.CurrentCommit=+git.$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)) +ifneq ($(strip $(LDFLAGS)),) + ldflags+=-extldflags=$(LDFLAGS) +endif + +GOFLAGS+=-ldflags="$(ldflags)" + +build: + rm -f migrate-lid + $(GOCC) build $(GOFLAGS) -o migrate-lid . +.PHONY: build diff --git a/cmd/migrate-lid/main.go b/cmd/migrate-lid/main.go new file mode 100644 index 000000000..4bd6ad21d --- /dev/null +++ b/cmd/migrate-lid/main.go @@ -0,0 +1,63 @@ +package main + +import ( + "os" + + "github.com/filecoin-project/boost/build" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" +) + +var log = logging.Logger("migrate") + +const ( + FlagBoostRepo = "boost-repo" +) + +var FlagRepo = &cli.StringFlag{ + Name: FlagBoostRepo, + EnvVars: []string{"BOOST_PATH"}, + Usage: "boost repo path", + Value: "~/.boost", +} + +var IsVeryVerbose bool + +var FlagVeryVerbose = &cli.BoolFlag{ + Name: "vv", + Usage: "enables very verbose mode, useful for debugging the CLI", + Destination: &IsVeryVerbose, +} + +func main() { + app := &cli.App{ + Name: "migrate-lid", + Usage: "Migrate boost to Local Index Directory", + EnableBashCompletion: true, + Version: build.UserVersion(), + Flags: []cli.Flag{ + FlagRepo, + FlagVeryVerbose, + }, + Commands: []*cli.Command{ + migrateLevelDBCmd, + migrateCouchDBCmd, + migrateReverseCmd, + }, + } + app.Setup() + + if err := app.Run(os.Args); err != nil { + os.Stderr.WriteString("Error: " + err.Error() + "\n") + } +} + +func before(cctx *cli.Context) error { + _ = logging.SetLogLevel("migrate", "INFO") + + if IsVeryVerbose { + _ = logging.SetLogLevel("migrate", "DEBUG") + } + + return nil +} diff --git a/cmd/migrate-lid/migrate_lid.go b/cmd/migrate-lid/migrate_lid.go new file mode 100644 index 000000000..f51892d72 --- /dev/null +++ b/cmd/migrate-lid/migrate_lid.go @@ -0,0 +1,868 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "sort" + "strings" + "time" + + "github.com/filecoin-project/boost/db" + "github.com/filecoin-project/boostd-data/couchbase" + "github.com/filecoin-project/boostd-data/ldb" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/go-address" + vfsm "github.com/filecoin-project/go-ds-versioning/pkg/fsm" + "github.com/filecoin-project/go-fil-markets/piecestore" + piecestoreimpl "github.com/filecoin-project/go-fil-markets/piecestore/impl" + "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-statemachine/fsm" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/node/repo" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + "github.com/ipld/go-car/v2/index" + "github.com/mitchellh/go-homedir" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" + "github.com/schollz/progressbar/v3" + "github.com/urfave/cli/v2" + "go.uber.org/zap" +) + +// The methods on the store that are used for migration +type StoreMigrationApi interface { + Start(ctx context.Context) error + IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) + AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error + AddDealForPiece(ctx context.Context, pcid cid.Cid, info model.DealInfo) error + ListPieces(ctx context.Context) ([]cid.Cid, error) + GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) +} + +var desc = "It is recommended to do the dagstore migration while boost is running. " + + "The dagstore migration may take several hours. It is safe to stop and restart " + + "the process. It will continue from where it was stopped.\n" + + "The pieceinfo migration must be done after boost has been shut down." + +func checkMigrateType(migrateType string) error { + if migrateType != "dagstore" && migrateType != "pieceinfo" { + return fmt.Errorf("invalid migration type '%s': must be either dagstore or pieceinfo", migrateType) + } + return nil +} + +var commonFlags = []cli.Flag{ + &cli.StringFlag{ + Name: "force", + Usage: "if the index has already been migrated, overwrite it", + Required: false, + }, +} + +var migrateLevelDBCmd = &cli.Command{ + Name: "leveldb", + Description: "Migrate boost piece information and dagstore to a leveldb store.\n" + desc, + Usage: "migrate-lid leveldb dagstore|pieceinfo", + Before: before, + Flags: commonFlags, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() == 0 { + return fmt.Errorf("must specify either dagstore or pieceinfo migration") + } + + // Get the type of migration (dagstore vs pieceinfo) + migrateType := cctx.Args().Get(0) + err := checkMigrateType(migrateType) + if err != nil { + return err + } + + // Create the leveldb directory if it doesn't already exist + repoDir, err := homedir.Expand(cctx.String(FlagBoostRepo)) + if err != nil { + return err + } + + repoPath, err := svc.MakeLevelDBDir(repoDir) + if err != nil { + return err + } + + // Create a connection to the leveldb store + store := ldb.NewStore(repoPath) + return migrate(cctx, "leveldb", store, migrateType) + }, +} + +var migrateCouchDBCmd = &cli.Command{ + Name: "couchbase", + Description: "Migrate boost piece information and dagstore to a couchbase store\n" + desc, + Usage: "migrate-lid couchbase dagstore|pieceinfo", + Before: before, + Flags: append(commonFlags, []cli.Flag{ + &cli.StringFlag{ + Name: "connect-string", + Usage: "couchbase connect string eg 'couchbase://127.0.0.1'", + Required: true, + }, + &cli.StringFlag{ + Name: "username", + Required: true, + }, + &cli.StringFlag{ + Name: "password", + Required: true, + }, + &cli.Uint64Flag{ + Name: "piece-meta-ram-quota-mb", + Usage: "megabytes of ram allocated to piece metadata couchbase bucket (recommended at least 1024)", + Value: 1024, + }, + &cli.Uint64Flag{ + Name: "mh-pieces-ram-quota-mb", + Usage: "megabytes of ram allocated to multihash to piece cid couchbase bucket (recommended at least 1024)", + Value: 1024, + }, + &cli.Uint64Flag{ + Name: "piece-offsets-ram-quota-mb", + Usage: "megabytes of ram allocated to piece offsets couchbase bucket (recommended at least 1024)", + Value: 1024, + }, + }...), + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() == 0 { + return fmt.Errorf("must specify either dagstore or pieceinfo migration") + } + + // Get the type of migration (dagstore vs pieceinfo) + migrateType := cctx.Args().Get(0) + err := checkMigrateType(migrateType) + if err != nil { + return err + } + + // Create a connection to the couchbase local index directory + settings := couchbase.DBSettings{ + ConnectString: cctx.String("connect-string"), + Auth: couchbase.DBSettingsAuth{ + Username: cctx.String("username"), + Password: cctx.String("password"), + }, + PieceMetadataBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: cctx.Uint64("piece-meta-ram-quota-mb"), + }, + MultihashToPiecesBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: cctx.Uint64("mh-pieces-ram-quota-mb"), + }, + PieceOffsetsBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: cctx.Uint64("piece-offsets-ram-quota-mb"), + }, + } + + store := couchbase.NewStore(settings) + return migrate(cctx, "couchbase", store, migrateType) + }, +} + +func migrate(cctx *cli.Context, dbType string, store StoreMigrationApi, migrateType string) error { + ctx := lcli.ReqContext(cctx) + svcCtx, cancel := context.WithCancel(ctx) + defer cancel() + + fmt.Println("Setting up connection and creating data structures... this might take a minute...") + err := store.Start(svcCtx) + if err != nil { + return fmt.Errorf("starting "+dbType+" store: %w", err) + } + + // Create a logger for the migration that outputs to a file in the + // current working directory + logPath := "migrate-" + dbType + ".log" + logger, err := createLogger(logPath) + if err != nil { + return err + } + + repoDir, err := homedir.Expand(cctx.String(FlagBoostRepo)) + if err != nil { + return err + } + + fmt.Print("Migrating to " + dbType + " Local Index Directory. ") + fmt.Println("See detailed logs of the migration at") + fmt.Println(logPath) + + // Create a progress bar + bar := progressbar.NewOptions(100, + progressbar.OptionEnableColorCodes(true), + progressbar.OptionFullWidth(), + progressbar.OptionSetPredictTime(true), + progressbar.OptionSetElapsedTime(false), + progressbar.OptionShowCount(), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "[green]=[reset]", + SaucerHead: "[green]>[reset]", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + })) + + if migrateType == "dagstore" { + // Migrate the indices + bar.Describe("Migrating indices...") + errCount, err := migrateIndices(ctx, logger, bar, repoDir, store, cctx.Bool("force")) + if errCount > 0 { + msg := fmt.Sprintf("Warning: there were errors migrating %d indices.", errCount) + msg += " See the log for details:\n" + logPath + fmt.Fprintf(os.Stderr, "\n"+msg+"\n") + } + if err != nil { + return fmt.Errorf("migrating indices: %w", err) + } + fmt.Println() + return nil + } + + // Migrate the piece store + bar.Describe("Migrating piece info...") + bar.Set(0) //nolint:errcheck + errCount, err := migratePieceStore(ctx, logger, bar, repoDir, store) + if errCount > 0 { + msg := fmt.Sprintf("Warning: there were errors migrating %d piece deal infos.", errCount) + msg += " See the log for details:\n" + logPath + fmt.Fprintf(os.Stderr, "\n"+msg+"\n") + } + if err != nil { + return fmt.Errorf("migrating piece store: %w", err) + } + fmt.Println() + return nil +} + +func migrateIndices(ctx context.Context, logger *zap.SugaredLogger, bar *progressbar.ProgressBar, repoDir string, store StoreMigrationApi, force bool) (int, error) { + indicesPath := path.Join(repoDir, "dagstore", "index") + logger.Infof("migrating dagstore indices at %s", indicesPath) + + idxPaths, err := getIndexPaths(indicesPath) + if err != nil { + return 0, err + } + + logger.Infof("starting migration of %d dagstore indices", len(idxPaths)) + bar.ChangeMax(len(idxPaths)) + + indicesStart := time.Now() + var count int + var errCount int + var indexTime time.Duration + for i, ipath := range idxPaths { + if ctx.Err() != nil { + return errCount, fmt.Errorf("index migration cancelled") + } + + start := time.Now() + + indexed, err := migrateIndex(ctx, ipath, store, force) + bar.Add(1) //nolint:errcheck + if err != nil { + logger.Errorw("migrate index failed", "piece cid", ipath.name, "err", err) + errCount++ + continue + } + + if indexed { + count++ + took := time.Since(start) + indexTime += took + logger.Infow("migrated index", "piece cid", ipath.name, "processed", i+1, "total", len(idxPaths), + "took", took.String(), "average", (indexTime / time.Duration(count)).String()) + } else { + logger.Infow("index already migrated", "piece cid", ipath.name, "processed", i+1, "total", len(idxPaths)) + } + } + + logger.Infow("migrated indices", "total", len(idxPaths), "took", time.Since(indicesStart).String()) + return errCount, nil +} + +func migrateIndex(ctx context.Context, ipath idxPath, store StoreMigrationApi, force bool) (bool, error) { + pieceCid, err := cid.Parse(ipath.name) + if err != nil { + return false, fmt.Errorf("parsing index name %s as cid: %w", ipath.name, err) + } + + if !force { + // Check if the index has already been migrated + isIndexed, err := store.IsIndexed(ctx, pieceCid) + if err != nil { + return false, fmt.Errorf("checking if index %s is already migrated: %w", ipath.path, err) + } + if isIndexed { + return false, nil + } + } + + // Load the index file + idx, err := loadIndex(ipath.path) + if err != nil { + return false, fmt.Errorf("loading index %s from disk: %w", ipath.path, err) + } + + itidx, ok := idx.(index.IterableIndex) + if !ok { + return false, fmt.Errorf("index %s is not iterable for piece %s", ipath.path, pieceCid) + } + + // Convert from IterableIndex to an array of records + records, err := getRecords(itidx) + if err != nil { + return false, fmt.Errorf("getting records for index %s: %w", ipath.path, err) + } + + // Add the index to the store + addStart := time.Now() + err = store.AddIndex(ctx, pieceCid, records, false) + if err != nil { + return false, fmt.Errorf("adding index %s to store: %w", ipath.path, err) + } + log.Debugw("AddIndex", "took", time.Since(addStart).String()) + + return true, nil +} + +func migratePieceStore(ctx context.Context, logger *zap.SugaredLogger, bar *progressbar.ProgressBar, repoDir string, store StoreMigrationApi) (int, error) { + // Open the datastore in the existing repo + ds, err := openDataStore(repoDir) + if err != nil { + return 0, fmt.Errorf("creating piece store from repo %s: %w", repoDir, err) + } + + // Get the miner address + maddr, err := modules.MinerAddress(ds) + if err != nil { + return 0, fmt.Errorf("getting miner address from repo %s: %w", repoDir, err) + } + + logger.Infof("migrating piece store deal information to Local Index Directory for miner %s", address.Address(maddr).String()) + start := time.Now() + + // Create a mapping of on-chain deal ID to deal proposal cid. + // This is needed below so that we can map from the legacy piece store + // info to a legacy deal. + propCidByChainDealID, err := getPropCidByChainDealID(ctx, ds) + if err != nil { + return 0, fmt.Errorf("building chain deal id -> proposal cid map: %w", err) + } + + ps, err := openPieceStore(ctx, ds) + if err != nil { + return 0, fmt.Errorf("opening piece store: %w", err) + } + + dbPath := path.Join(repoDir, "boost.db?cache=shared") + sqldb, err := db.SqlDB(dbPath) + if err != nil { + return 0, fmt.Errorf("opening boost sqlite db: %w", err) + } + + qry := "SELECT ID, ChainDealID FROM Deals" + rows, err := sqldb.QueryContext(ctx, qry) + if err != nil { + return 0, fmt.Errorf("executing select on Deals: %w", err) + } + + boostDeals := make(map[abi.DealID]string) + + for rows.Next() { + var uuid string + var chainDealId abi.DealID + + err := rows.Scan(&uuid, &chainDealId) + if err != nil { + return 0, fmt.Errorf("executing row scan: %w", err) + } + + boostDeals[chainDealId] = uuid + } + + pcids, err := ps.ListPieceInfoKeys() + if err != nil { + return 0, fmt.Errorf("getting piece store keys: %w", err) + } + + // Ensure the same order in case the import is stopped and restarted + sort.Slice(pcids, func(i, j int) bool { + return pcids[0].String() < pcids[1].String() + }) + + logger.Infof("starting migration of %d piece infos", len(pcids)) + bar.ChangeMax(len(pcids)) + + var indexTime time.Duration + var count int + var errorCount int + for i, pcid := range pcids { + bar.Add(1) //nolint:errcheck + + pieceStart := time.Now() + + pi, err := ps.GetPieceInfo(pcid) + if err != nil { + errorCount++ + logger.Errorw("cant get piece info for piece", "pcid", pcid, "err", err) + continue + } + + var addedDeals bool + for _, d := range pi.Deals { + // Find the deal corresponding to the deal info's DealID + proposalCid, okLegacy := propCidByChainDealID[d.DealID] + uuid, okBoost := boostDeals[d.DealID] + + if !okLegacy && !okBoost { + logger.Errorw("cant find boost deal or legacy deal for piece", + "pcid", pcid, "chain-deal-id", d.DealID, "err", err) + continue + } + + isLegacy := false + if uuid == "" { + uuid = proposalCid.String() + isLegacy = true + } + + dealInfo := model.DealInfo{ + DealUuid: uuid, + IsLegacy: isLegacy, + ChainDealID: d.DealID, + MinerAddr: address.Address(maddr), + SectorID: d.SectorID, + PieceOffset: d.Offset, + PieceLength: d.Length, + } + + err = store.AddDealForPiece(ctx, pcid, dealInfo) + if err == nil { + addedDeals = true + } else { + logger.Errorw("cant add deal info for piece", "pcid", pcid, "chain-deal-id", d.DealID, "err", err) + } + } + + if addedDeals { + count++ + } else { + errorCount++ + } + took := time.Since(pieceStart) + indexTime += took + avgDenom := count + if avgDenom == 0 { + avgDenom = 1 + } + logger.Infow("migrated piece deals", "piece cid", pcid, "processed", i+1, "total", len(pcids), + "took", took.String(), "average", (indexTime / time.Duration(avgDenom)).String()) + } + + logger.Infow("migrated piece deals", "count", len(pcids), "errors", errorCount, "took", time.Since(start)) + + return errorCount, nil +} + +func getPropCidByChainDealID(ctx context.Context, ds *backupds.Datastore) (map[abi.DealID]cid.Cid, error) { + deals, err := getLegacyDealsFSM(ctx, ds) + if err != nil { + return nil, err + } + + // Build a mapping of chain deal ID to proposal CID + var list []storagemarket.MinerDeal + if err := deals.List(&list); err != nil { + return nil, err + } + + byChainDealID := make(map[abi.DealID]cid.Cid, len(list)) + for _, d := range list { + if d.DealID != 0 { + byChainDealID[d.DealID] = d.ProposalCid + } + } + + return byChainDealID, nil +} + +func getLegacyDealsFSM(ctx context.Context, ds *backupds.Datastore) (fsm.Group, error) { + // Get the deals FSM + provDS := namespace.Wrap(ds, datastore.NewKey("/deals/provider")) + deals, migrate, err := vfsm.NewVersionedFSM(provDS, fsm.Parameters{ + StateType: storagemarket.MinerDeal{}, + StateKeyField: "State", + }, nil, "2") + if err != nil { + return nil, fmt.Errorf("reading legacy deals from datastore: %w", err) + } + + err = migrate(ctx) + if err != nil { + return nil, fmt.Errorf("running provider fsm migration script: %w", err) + } + + return deals, err +} + +func openDataStore(path string) (*backupds.Datastore, error) { + ctx := context.Background() + + rpo, err := repo.NewFS(path) + if err != nil { + return nil, fmt.Errorf("could not open repo %s: %w", path, err) + } + + exists, err := rpo.Exists() + if err != nil { + return nil, fmt.Errorf("checking repo %s exists: %w", path, err) + } + if !exists { + return nil, fmt.Errorf("repo does not exist: %s", path) + } + + lr, err := rpo.Lock(repo.StorageMiner) + if err != nil { + return nil, fmt.Errorf("locking repo %s: %w", path, err) + } + + mds, err := lr.Datastore(ctx, "/metadata") + if err != nil { + return nil, err + } + + bds, err := backupds.Wrap(mds, "") + if err != nil { + return nil, fmt.Errorf("opening backupds: %w", err) + } + + return bds, nil +} + +func getRecords(subject index.Index) ([]model.Record, error) { + records := make([]model.Record, 0) + + switch idx := subject.(type) { + case index.IterableIndex: + err := idx.ForEach(func(m multihash.Multihash, offset uint64) error { + + cid := cid.NewCidV1(cid.Raw, m) + + records = append(records, model.Record{ + Cid: cid, + OffsetSize: model.OffsetSize{ + Offset: offset, + Size: 0, + }, + }) + + return nil + }) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("wanted %v but got %v\n", multicodec.CarMultihashIndexSorted, idx.Codec()) + } + return records, nil +} + +type idxPath struct { + name string + path string +} + +func getIndexPaths(pathDir string) ([]idxPath, error) { + files, err := ioutil.ReadDir(pathDir) + if err != nil { + return nil, err + } + + idxPaths := make([]idxPath, 0, len(files)) + for _, f := range files { + name := f.Name() + + if strings.Contains(name, "full.idx") { + filepath := pathDir + "/" + name + name = strings.ReplaceAll(name, ".full.idx", "") + + idxPaths = append(idxPaths, idxPath{ + name: name, + path: filepath, + }) + } + } + + return idxPaths, nil +} + +func loadIndex(path string) (index.Index, error) { + defer func(now time.Time) { + log.Debugw("loadindex", "took", time.Since(now)) + }(time.Now()) + + idxf, err := os.Open(path) + if err != nil { + return nil, err + } + defer idxf.Close() + + subject, err := index.ReadFrom(idxf) + if err != nil { + return nil, err + } + + return subject, nil +} + +var migrateReverseCmd = &cli.Command{ + Name: "reverse", + Usage: "Do a reverse migration from the local index directory back to the legacy format", + Subcommands: []*cli.Command{ + migrateReverseLeveldbCmd, + migrateReverseCouchbaseCmd, + }, +} + +var migrateReverseLeveldbCmd = &cli.Command{ + Name: "leveldb", + Usage: "Reverse migrate a leveldb local index directory", + Before: before, + Action: func(cctx *cli.Context) error { + return migrateReverse(cctx, "leveldb") + }, +} + +var migrateReverseCouchbaseCmd = &cli.Command{ + Name: "couchbase", + Usage: "Reverse migrate a couchbase local index directory", + Before: before, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "connect-string", + Usage: "couchbase connect string eg 'couchbase://127.0.0.1'", + Required: true, + }, + &cli.StringFlag{ + Name: "username", + Required: true, + }, + &cli.StringFlag{ + Name: "password", + Required: true, + }, + }, + Action: func(cctx *cli.Context) error { + return migrateReverse(cctx, "couchbase") + }, +} + +func migrateReverse(cctx *cli.Context, dbType string) error { + // Create a logger for the migration that outputs to a file in the + // current working directory + logPath := "reverse-migrate-" + dbType + ".log" + logger, err := createLogger(logPath) + if err != nil { + return err + } + fmt.Printf("Performing %s reverse migration with logs at %s\n", dbType, logPath) + + repoDir, err := homedir.Expand(cctx.String(FlagBoostRepo)) + if err != nil { + return err + } + + // Get a leveldb / couchbase store + var store StoreMigrationApi + if dbType == "leveldb" { + // Create a connection to the leveldb store + ldbRepoPath, err := svc.MakeLevelDBDir(repoDir) + if err != nil { + return err + } + store = ldb.NewStore(ldbRepoPath) + } else { + // Create a connection to the couchbase local index directory + settings := couchbase.DBSettings{ + ConnectString: cctx.String("connect-string"), + Auth: couchbase.DBSettingsAuth{ + Username: cctx.String("username"), + Password: cctx.String("password"), + }, + } + + store = couchbase.NewStore(settings) + } + + // Perform the reverse migration + err = migrateDBReverse(cctx, repoDir, dbType, store, logger) + if err != nil { + return err + } + + fmt.Println("Reverse migration complete") + return nil +} + +func migrateDBReverse(cctx *cli.Context, repoDir string, dbType string, pieceDir StoreMigrationApi, logger *zap.SugaredLogger) error { + ctx := lcli.ReqContext(cctx) + start := time.Now() + + svcCtx, cancel := context.WithCancel(ctx) + defer cancel() + err := pieceDir.Start(svcCtx) + if err != nil { + return fmt.Errorf("starting "+dbType+" store: %w", err) + } + + pcids, err := pieceDir.ListPieces(ctx) + if err != nil { + return fmt.Errorf("listing local index directory pieces: %w", err) + } + + logger.Infof("starting migration of %d piece infos from %s local index directory to piece store", len(pcids), dbType) + + // Open the datastore + ds, err := openDataStore(repoDir) + if err != nil { + return fmt.Errorf("creating datastore from repo %s: %w", repoDir, err) + } + + // Open the Piece Store + ps, err := openPieceStore(ctx, ds) + if err != nil { + return fmt.Errorf("opening piece store: %w", err) + } + + // For each piece in the local index directory + var errorCount int + for i, pieceCid := range pcids { + // Reverse migrate the piece + migrated, err := migrateReversePiece(ctx, pieceCid, pieceDir, ps) + if err != nil { + errorCount++ + logger.Errorw("failed to reverse migrate piece", "pieceCid", pieceCid, "index", i, "total", len(pcids), "error", err) + } else if migrated > 0 { + logger.Infow("reverse migrated piece", "pieceCid", pieceCid, "index", i, "total", len(pcids), "migrated-deals", migrated) + } else { + logger.Infow("no deals to migrate for piece", "pieceCid", pieceCid, "index", i, "total", len(pcids)) + } + } + + logger.Infow("reverse migration complete", "count", len(pcids), "errors", errorCount, "took", time.Since(start)) + return nil +} + +func migrateReversePiece(ctx context.Context, pieceCid cid.Cid, pieceDir StoreMigrationApi, ps piecestore.PieceStore) (int, error) { + // Get the piece metadata from the local index directory + pieceDirPieceInfo, err := pieceDir.GetPieceMetadata(ctx, pieceCid) + if err != nil { + return 0, fmt.Errorf("getting piece metadata for piece %s", pieceCid) + } + + // Get the deals from the piece metadata + var pieceStoreDeals []piecestore.DealInfo + pieceStorePieceInfo, err := ps.GetPieceInfo(pieceCid) + if err != nil { + if !errors.Is(err, retrievalmarket.ErrNotFound) { + return 0, fmt.Errorf("getting piece info from piece store for piece %s", pieceCid) + } + } else { + pieceStoreDeals = pieceStorePieceInfo.Deals + } + + // Iterate over each local index directory deal and add it to the piece store + // if it's not there already + var migrated int + for _, pieceDirDeal := range pieceDirPieceInfo.Deals { + // Check if the local index directory deal is already in the piece store + var has bool + for _, pieceStoreDeal := range pieceStoreDeals { + if pieceStoreDeal.SectorID == pieceDirDeal.SectorID && + pieceStoreDeal.Offset == pieceDirDeal.PieceOffset && + pieceStoreDeal.Length == pieceDirDeal.PieceLength { + + has = true + } + } + if has { + continue + } + + // The piece store doesn't yet have the deal, so add it + newDealInfo := piecestore.DealInfo{ + DealID: pieceDirDeal.ChainDealID, + SectorID: pieceDirDeal.SectorID, + Offset: pieceDirDeal.PieceOffset, + Length: pieceDirDeal.PieceLength, + } + + // Note: the second parameter is ignored by the piece store + // implementation + err = ps.AddDealForPiece(pieceCid, cid.Undef, newDealInfo) + if err != nil { + return 0, fmt.Errorf("adding deal to piece store for piece %s: %w", pieceCid, err) + } + + migrated++ + } + + return migrated, nil +} + +func openPieceStore(ctx context.Context, ds *backupds.Datastore) (piecestore.PieceStore, error) { + // Open the piece store + ps, err := piecestoreimpl.NewPieceStore(namespace.Wrap(ds, datastore.NewKey("/storagemarket"))) + if err != nil { + return nil, fmt.Errorf("creating piece store from datastore : %w", err) + } + + // Wait for the piece store to be ready + ch := make(chan error, 1) + ps.OnReady(func(e error) { + ch <- e + }) + + err = ps.Start(ctx) + if err != nil { + return nil, fmt.Errorf("starting piece store: %w", err) + } + + select { + case err = <-ch: + if err != nil { + return nil, fmt.Errorf("waiting for piece store to be ready: %w", err) + } + case <-ctx.Done(): + return nil, errors.New("cancelled while waiting for piece store to be ready") + } + + return ps, nil +} + +func createLogger(logPath string) (*zap.SugaredLogger, error) { + logCfg := zap.NewDevelopmentConfig() + logCfg.OutputPaths = []string{logPath} + zl, err := logCfg.Build() + if err != nil { + return nil, err + } + defer zl.Sync() //nolint:errcheck + return zl.Sugar(), err +} diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md index b752e996a..538b60902 100644 --- a/documentation/en/api-v1-methods.md +++ b/documentation/en/api-v1-methods.md @@ -21,6 +21,7 @@ * [BoostDealBySignedProposalCid](#boostdealbysignedproposalcid) * [BoostDummyDeal](#boostdummydeal) * [BoostIndexerAnnounceAllDeals](#boostindexerannouncealldeals) + * [BoostIndexerListMultihashes](#boostindexerlistmultihashes) * [BoostMakeDeal](#boostmakedeal) * [BoostOfflineDealWithData](#boostofflinedealwithdata) * [Deals](#deals) @@ -82,12 +83,9 @@ * [NetStat](#netstat) * [Online](#online) * [OnlineBackup](#onlinebackup) -* [Pieces](#pieces) - * [PiecesGetCIDInfo](#piecesgetcidinfo) - * [PiecesGetMaxOffset](#piecesgetmaxoffset) - * [PiecesGetPieceInfo](#piecesgetpieceinfo) - * [PiecesListCidInfos](#pieceslistcidinfos) - * [PiecesListPieces](#pieceslistpieces) +* [Pd](#pd) + * [PdBuildIndexForPieceCid](#pdbuildindexforpiececid) + * [PdMarkIndexErrored](#pdmarkindexerrored) * [Runtime](#runtime) * [RuntimeSubsystems](#runtimesubsystems) * [Sectors](#sectors) @@ -537,6 +535,27 @@ Inputs: `null` Response: `{}` +### BoostIndexerListMultihashes + + +Perms: admin + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: +```json +[ + "Bw==" +] +``` + ### BoostMakeDeal @@ -1724,45 +1743,13 @@ Inputs: Response: `{}` -## Pieces - - -### PiecesGetCIDInfo +## Pd -Perms: read - -Inputs: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "CID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - }, - "PieceBlockLocations": [ - { - "RelOffset": 42, - "BlockSize": 42, - "PieceCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } - } - ] -} -``` - -### PiecesGetMaxOffset - +### PdBuildIndexForPieceCid +There are not yet any comments for this method. -Perms: read +Perms: admin Inputs: ```json @@ -1773,70 +1760,24 @@ Inputs: ] ``` -Response: `42` +Response: `{}` -### PiecesGetPieceInfo +### PdMarkIndexErrored -Perms: read +Perms: admin Inputs: ```json [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` - -Response: -```json -{ - "PieceCID": { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" }, - "Deals": [ - { - "DealID": 5432, - "SectorID": 9, - "Offset": 1032, - "Length": 1032 - } - ] -} -``` - -### PiecesListCidInfos - - -Perms: read - -Inputs: `null` - -Response: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } + "string value" ] ``` -### PiecesListPieces - - -Perms: read - -Inputs: `null` - -Response: -```json -[ - { - "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" - } -] -``` +Response: `{}` ## Runtime diff --git a/extern/boostd-data/Makefile b/extern/boostd-data/Makefile new file mode 100644 index 000000000..446fe7077 --- /dev/null +++ b/extern/boostd-data/Makefile @@ -0,0 +1,17 @@ +SHELL=/usr/bin/env bash + +unexport GOFLAGS + +GOCC?=go + +ldflags=-X=github.com/filecoin-project/boost/build.CurrentCommit=+git.$(subst -,.,$(shell git describe --always --match=NeVeRmAtCh --dirty 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)) +ifneq ($(strip $(LDFLAGS)),) + ldflags+=-extldflags=$(LDFLAGS) +endif + +GOFLAGS+=-ldflags="$(ldflags)" + +build: + rm -f boostd-data + $(GOCC) build $(GOFLAGS) -o boostd-data ./cmd +.PHONY: build diff --git a/extern/boostd-data/client/client.go b/extern/boostd-data/client/client.go index 5939c12b9..e41a7952d 100644 --- a/extern/boostd-data/client/client.go +++ b/extern/boostd-data/client/client.go @@ -1,10 +1,12 @@ package client import ( + "context" + "fmt" "time" - "github.com/ethereum/go-ethereum/rpc" - "github.com/filecoin-project/boost/cmd/boostd-data/model" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/go-jsonrpc" "github.com/ipfs/go-cid" logger "github.com/ipfs/go-log/v2" "github.com/ipld/go-car/v2/index" @@ -14,33 +16,62 @@ import ( var log = logger.Logger("boostd-data-client") type Store struct { - client *rpc.Client + client struct { + AddDealForPiece func(context.Context, cid.Cid, model.DealInfo) error + AddIndex func(context.Context, cid.Cid, []model.Record, bool) error + IsIndexed func(ctx context.Context, pieceCid cid.Cid) (bool, error) + IsCompleteIndex func(ctx context.Context, pieceCid cid.Cid) (bool, error) + GetIndex func(context.Context, cid.Cid) ([]model.Record, error) + GetOffsetSize func(context.Context, cid.Cid, mh.Multihash) (*model.OffsetSize, error) + ListPieces func(ctx context.Context) ([]cid.Cid, error) + GetPieceMetadata func(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) + GetPieceDeals func(context.Context, cid.Cid) ([]model.DealInfo, error) + SetCarSize func(ctx context.Context, pieceCid cid.Cid, size uint64) error + MarkIndexErrored func(context.Context, cid.Cid, string) error + IndexedAt func(context.Context, cid.Cid) (time.Time, error) + PiecesContainingMultihash func(context.Context, mh.Multihash) ([]cid.Cid, error) + RemoveDealForPiece func(context.Context, cid.Cid, string) error + RemovePieceMetadata func(context.Context, cid.Cid) error + RemoveIndexes func(context.Context, cid.Cid) error + NextPiecesToCheck func(ctx context.Context) ([]cid.Cid, error) + FlagPiece func(ctx context.Context, pieceCid cid.Cid) error + UnflagPiece func(ctx context.Context, pieceCid cid.Cid) error + FlaggedPiecesList func(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) + FlaggedPiecesCount func(ctx context.Context) (int, error) + } + closer jsonrpc.ClientCloser +} + +func NewStore() *Store { + return &Store{} } -func NewStore(addr string) (*Store, error) { - client, err := rpc.Dial(addr) +func (s *Store) Dial(ctx context.Context, addr string) error { + var err error + s.closer, err = jsonrpc.NewClient(ctx, addr, "boostddata", &s.client, nil) if err != nil { - return nil, err + return fmt.Errorf("dialing local index directory server: %w", err) } + return nil +} - return &Store{ - client: client, - }, nil +func (s *Store) Close(_ context.Context) { + if s != nil && s.closer != nil { + s.closer() + } } -func (s *Store) GetIndex(pieceCid cid.Cid) (index.Index, error) { - var resp []model.Record - err := s.client.Call(&resp, "boostddata_getIndex", pieceCid) +func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) (index.Index, error) { + resp, err := s.client.GetIndex(ctx, pieceCid) if err != nil { return nil, err } - //TODO: figure out how to remove this conversion var records []index.Record for _, r := range resp { records = append(records, index.Record{ - r.Cid, - r.Offset, + Cid: r.Cid, + Offset: r.Offset, }) } @@ -53,73 +84,95 @@ func (s *Store) GetIndex(pieceCid cid.Cid) (index.Index, error) { return &mis, nil } -func (s *Store) GetRecords(pieceCid cid.Cid) ([]model.Record, error) { - var resp []model.Record - err := s.client.Call(&resp, "boostddata_getIndex", pieceCid) +func (s *Store) GetRecords(ctx context.Context, pieceCid cid.Cid) ([]model.Record, error) { + resp, err := s.client.GetIndex(ctx, pieceCid) if err != nil { return nil, err } - log.Warnw("get-index", "piece-cid", pieceCid, "records", len(resp)) + log.Debugw("get-records", "piece-cid", pieceCid, "records", len(resp)) return resp, nil } -func (s *Store) GetPieceDeals(pieceCid cid.Cid) ([]model.DealInfo, error) { - var resp []model.DealInfo - err := s.client.Call(&resp, "boostddata_getPieceDeals", pieceCid) - if err != nil { - return nil, err - } +func (s *Store) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { + return s.client.GetPieceMetadata(ctx, pieceCid) +} - return resp, nil +func (s *Store) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) { + return s.client.GetPieceDeals(ctx, pieceCid) } -func (s *Store) PiecesContaining(m mh.Multihash) ([]cid.Cid, error) { - var resp []cid.Cid - err := s.client.Call(&resp, "boostddata_piecesContainingMultihash", m) - if err != nil { - return nil, err - } +func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ([]cid.Cid, error) { + return s.client.PiecesContainingMultihash(ctx, m) +} - return resp, nil +func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error { + return s.client.MarkIndexErrored(ctx, pieceCid, err) } -func (s *Store) AddDealForPiece(pieceCid cid.Cid, dealInfo model.DealInfo) error { - return s.client.Call(nil, "boostddata_addDealForPiece", pieceCid, dealInfo) +func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { + return s.client.AddDealForPiece(ctx, pieceCid, dealInfo) } -func (s *Store) AddIndex(pieceCid cid.Cid, records []model.Record) error { - log.Warnw("add-index", "piece-cid", pieceCid, "records", len(records)) +func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { + return s.client.SetCarSize(ctx, pieceCid, size) +} - return s.client.Call(nil, "boostddata_addIndex", pieceCid, records) +func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error { + log.Debugw("add-index", "piece-cid", pieceCid, "records", len(records)) + + return s.client.AddIndex(ctx, pieceCid, records, isCompleteIndex) } -func (s *Store) IsIndexed(pieceCid cid.Cid) (bool, error) { - var t time.Time +func (s *Store) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) { + return s.client.IsIndexed(ctx, pieceCid) +} - err := s.client.Call(&t, "boostddata_indexedAt", pieceCid) - if err != nil { - return false, err - } - return !t.IsZero(), nil +func (s *Store) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, error) { + return s.client.IsCompleteIndex(ctx, pieceCid) } -func (s *Store) IndexedAt(pieceCid cid.Cid) (time.Time, error) { - var ts time.Time - err := s.client.Call(&ts, "boostddata_indexedAt", pieceCid) - if err != nil { - return time.Time{}, err - } - return ts, nil +func (s *Store) IndexedAt(ctx context.Context, pieceCid cid.Cid) (time.Time, error) { + return s.client.IndexedAt(ctx, pieceCid) } -func (s *Store) GetOffset(pieceCid cid.Cid, hash mh.Multihash) (uint64, error) { - var resp uint64 - err := s.client.Call(&resp, "boostddata_getOffset", pieceCid, hash) - if err != nil { - return 0, err - } +func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { + return s.client.GetOffsetSize(ctx, pieceCid, hash) +} - return resp, nil +func (s *Store) RemoveDealForPiece(ctx context.Context, pieceCid cid.Cid, dealId string) error { + return s.client.RemoveDealForPiece(ctx, pieceCid, dealId) +} + +func (s *Store) RemovePieceMetadata(ctx context.Context, pieceCid cid.Cid) error { + return s.client.RemovePieceMetadata(ctx, pieceCid) +} + +func (s *Store) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { + return s.client.RemoveIndexes(ctx, pieceCid) +} + +func (s *Store) ListPieces(ctx context.Context) ([]cid.Cid, error) { + return s.client.ListPieces(ctx) +} + +func (s *Store) NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) { + return s.client.NextPiecesToCheck(ctx) +} + +func (s *Store) FlagPiece(ctx context.Context, pieceCid cid.Cid) error { + return s.client.FlagPiece(ctx, pieceCid) +} + +func (s *Store) UnflagPiece(ctx context.Context, pieceCid cid.Cid) error { + return s.client.UnflagPiece(ctx, pieceCid) +} + +func (s *Store) FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) { + return s.client.FlaggedPiecesList(ctx, cursor, offset, limit) +} + +func (s *Store) FlaggedPiecesCount(ctx context.Context) (int, error) { + return s.client.FlaggedPiecesCount(ctx) } diff --git a/extern/boostd-data/cmd/main.go b/extern/boostd-data/cmd/main.go new file mode 100644 index 000000000..d53109e5c --- /dev/null +++ b/extern/boostd-data/cmd/main.go @@ -0,0 +1,43 @@ +package main + +import ( + "github.com/filecoin-project/boostd-data/shared/cliutil" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" + "os" +) + +var log = logging.Logger("boostd-data") + +// This service exposes an RPC API that is called by boostd in order to manage +// boostd's data. The service provides different implementations of the data +// interface according to the technologies that a user chooses (eg leveldb vs +// couchbase) +func main() { + app := &cli.App{ + Name: "boostd-data", + Usage: "Service that implements boostd data API", + EnableBashCompletion: true, + Flags: []cli.Flag{ + cliutil.FlagVeryVerbose, + }, + Commands: []*cli.Command{ + runCmd, + }, + } + app.Setup() + + if err := app.Run(os.Args); err != nil { + os.Stderr.WriteString("Error: " + err.Error() + "\n") + } +} + +func before(cctx *cli.Context) error { + _ = logging.SetLogLevel("boostd-data", "INFO") + + if cliutil.IsVeryVerbose { + _ = logging.SetLogLevel("boostd-data", "DEBUG") + } + + return nil +} diff --git a/extern/boostd-data/cmd/run.go b/extern/boostd-data/cmd/run.go new file mode 100644 index 000000000..132884120 --- /dev/null +++ b/extern/boostd-data/cmd/run.go @@ -0,0 +1,158 @@ +package main + +import ( + "context" + "fmt" + "github.com/filecoin-project/boostd-data/couchbase" + "github.com/filecoin-project/boostd-data/shared/cliutil" + "github.com/filecoin-project/boostd-data/shared/tracing" + "github.com/filecoin-project/boostd-data/svc" + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" + "net/http" +) + +var runCmd = &cli.Command{ + Name: "run", + Subcommands: []*cli.Command{ + leveldbCmd, + couchbaseCmd, + }, +} + +var runFlags = []cli.Flag{ + &cli.UintFlag{ + Name: "port", + Usage: "the port the boostd-data listens on", + Value: 8042, + }, + &cli.BoolFlag{ + Name: "pprof", + Usage: "run pprof web server on localhost:6071", + }, + &cli.BoolFlag{ + Name: "tracing", + Usage: "enables tracing of boostd-data calls", + Value: false, + }, + &cli.StringFlag{ + Name: "tracing-endpoint", + Usage: "the endpoint for the tracing exporter", + Value: "http://tempo:14268/api/traces", + }, +} + +var leveldbCmd = &cli.Command{ + Name: "leveldb", + Usage: "Run boostd-data with a leveldb database", + Before: before, + Flags: append([]cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Usage: "repo directory where the leveldb database is created", + Value: "~/.boost", + }}, + runFlags..., + ), + Action: func(cctx *cli.Context) error { + repoDir, err := homedir.Expand(cctx.String("repo")) + if err != nil { + return err + } + + // Create a leveldb data service + dbsvc, err := svc.NewLevelDB(repoDir) + if err != nil { + return err + } + + return runAction(cctx, "leveldb", dbsvc) + }, +} + +var couchbaseCmd = &cli.Command{ + Name: "couchbase", + Usage: "Run boostd-data with a couchbase database", + Before: before, + Flags: append([]cli.Flag{ + &cli.StringFlag{ + Name: "connect-string", + Usage: "couchbase connect string eg 'couchbase://127.0.0.1'", + Required: true, + }, + &cli.StringFlag{ + Name: "username", + Required: true, + }, + &cli.StringFlag{ + Name: "password", + Required: true, + }}, + runFlags..., + ), + Action: func(cctx *cli.Context) error { + // Create a couchbase data service + settings := couchbase.DBSettings{ + ConnectString: cctx.String("connect-string"), + Auth: couchbase.DBSettingsAuth{ + Username: cctx.String("username"), + Password: cctx.String("password"), + }, + } + + bdsvc := svc.NewCouchbase(settings) + return runAction(cctx, "couchbase", bdsvc) + }, +} + +func runAction(cctx *cli.Context, dbType string, store *svc.Service) error { + ctx := cliutil.ReqContext(cctx) + + if cctx.Bool("pprof") { + go func() { + err := http.ListenAndServe("localhost:6071", nil) + if err != nil { + log.Error(err) + } + }() + } + + // Instantiate the tracer and exporter + enableTracing := cctx.Bool("tracing") + var tracingStopper func(context.Context) error + var err error + if enableTracing { + tracingStopper, err = tracing.New("boostd-data", cctx.String("tracing-endpoint")) + if err != nil { + return fmt.Errorf("failed to instantiate tracer: %w", err) + } + log.Info("Tracing exporter enabled") + } + + // Start the server + port := cctx.Int("port") + err = store.Start(ctx, port) + if err != nil { + return fmt.Errorf("starting %s store: %w", dbType, err) + } + + log.Infof("Started boostd-data %s service on port %d", + dbType, port) + + // Monitor for shutdown. + <-ctx.Done() + + log.Info("Graceful shutdown successful") + + // Sync all loggers. + _ = log.Sync() //nolint:errcheck + + if enableTracing { + err = tracingStopper(ctx) + if err != nil { + return err + } + } + + return nil +} diff --git a/extern/boostd-data/couchbase/db.go b/extern/boostd-data/couchbase/db.go index 182e810dc..18ce5793b 100644 --- a/extern/boostd-data/couchbase/db.go +++ b/extern/boostd-data/couchbase/db.go @@ -2,301 +2,1178 @@ package couchbase import ( "context" - "encoding/binary" - "encoding/json" "errors" "fmt" "time" "github.com/couchbase/gocb/v2" - "github.com/filecoin-project/boost/cmd/boostd-data/model" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" - ds "github.com/ipfs/go-datastore" - carindex "github.com/ipld/go-car/v2/index" "github.com/multiformats/go-multihash" + "go.opentelemetry.io/otel/attribute" + "golang.org/x/sync/errgroup" ) -var ( - // LevelDB key value for storing next free cursor. - keyNextCursor uint64 = 0 - dskeyNextCursor datastore.Key +const pieceDirBucketPrefix = "piece-dir." +const pieceCidToMetadataBucket = pieceDirBucketPrefix + "piece-metadata" +const multihashToPiecesBucket = pieceDirBucketPrefix + "mh-to-pieces" +const pieceOffsetsBucket = pieceDirBucketPrefix + "piece-offsets" +const metaBucket = pieceDirBucketPrefix + "metadata" - // LevelDB key prefix for PieceCid to cursor table. - // LevelDB keys will be built by concatenating PieceCid to this prefix. - prefixPieceCidToCursor uint64 = 1 - sprefixPieceCidToCursor string +// The maximum length for a couchbase key is 250 bytes, but we don't need a +// key that long, 128 bytes is more than enough for unique keys +const maxCouchKeyLen = 128 - // LevelDB key prefix for Multihash to PieceCids table. - // LevelDB keys will be built by concatenating Multihash to this prefix. - prefixMhtoPieceCids uint64 = 2 - sprefixMhtoPieceCids string +// maxCasRetries is the number of times to retry an update operation when +// there is a cas mismatch +const maxCasRetries = 10 - size = binary.MaxVarintLen64 -) +// The current piece metadata version. This version will be used when doing +// data migrations (migrations are not yet implemented in version 1). +const pieceMetadataVersion = "1" -func init() { - buf := make([]byte, size) - binary.PutUvarint(buf, keyNextCursor) - dskeyNextCursor = datastore.NewKey(string(buf)) +func newCouchbaseMetadata() CouchbaseMetadata { + return CouchbaseMetadata{ + Metadata: model.Metadata{ + Version: pieceMetadataVersion, + }, + } +} - buf = make([]byte, size) - binary.PutUvarint(buf, prefixPieceCidToCursor) - sprefixPieceCidToCursor = string(buf) +// Used to store values as binary (rather than json) +var binaryTranscoder = gocb.NewRawBinaryTranscoder() - buf = make([]byte, size) - binary.PutUvarint(buf, prefixMhtoPieceCids) - sprefixMhtoPieceCids = string(buf) +type DB struct { + settings DBSettings + cluster *gocb.Cluster + pcidToMeta *gocb.Collection + mhToPieces *gocb.Collection + pieceOffsets *gocb.Collection } -type DB struct { - col *gocb.Collection +type DBSettingsAuth struct { + Username string + Password string } -func newDB() (*DB, error) { - bucketName := "piecestore" - username := "Administrator" - password := "boostdemo" +type DBSettingsBucket struct { + RAMQuotaMB uint64 +} + +type DBSettings struct { + ConnectString string + Auth DBSettingsAuth + PieceMetadataBucket DBSettingsBucket + MultihashToPiecesBucket DBSettingsBucket + PieceOffsetsBucket DBSettingsBucket + TestMode bool +} - cluster, err := gocb.Connect("couchbase://127.0.0.1", gocb.ClusterOptions{ +const connectTimeout = 5 * time.Second +const kvTimeout = 30 * time.Second + +func newDB(ctx context.Context, settings DBSettings) (*DB, error) { + cluster, err := gocb.Connect(settings.ConnectString, gocb.ClusterOptions{ + TimeoutsConfig: gocb.TimeoutsConfig{ + ConnectTimeout: connectTimeout, + KVTimeout: kvTimeout, + }, Authenticator: gocb.PasswordAuthenticator{ - Username: username, - Password: password, + Username: settings.Auth.Username, + Password: settings.Auth.Password, }, }) + if err != nil { + return nil, fmt.Errorf("connecting to couchbase cluster %s: %w", settings.ConnectString, err) + } + + pingStart := time.Now() + err = pingCluster(ctx, cluster, settings.ConnectString) if err != nil { return nil, err } - bucket := cluster.Bucket(bucketName) + log.Infow("Connected to couchbase cluster", + "connect-string", settings.ConnectString, + "max-service-latency", time.Since(pingStart).String()) - err = bucket.WaitUntilReady(5*time.Second, nil) + // Set up the buckets + db := &DB{settings: settings, cluster: cluster} + pcidToMeta, err := CreateBucket(ctx, cluster, pieceCidToMetadataBucket, settings.PieceMetadataBucket.RAMQuotaMB) + if err != nil { + return nil, fmt.Errorf("Creating bucket %s for couchbase server %s: %w", pieceCidToMetadataBucket, settings.ConnectString, err) + } + db.pcidToMeta = pcidToMeta.DefaultCollection() + + mhToPieces, err := CreateBucket(ctx, cluster, multihashToPiecesBucket, settings.MultihashToPiecesBucket.RAMQuotaMB) + if err != nil { + return nil, fmt.Errorf("Creating bucket %s for couchbase server %s: %w", multihashToPiecesBucket, settings.ConnectString, err) + } + db.mhToPieces = mhToPieces.DefaultCollection() + + pieceOffsets, err := CreateBucket(ctx, cluster, pieceOffsetsBucket, settings.PieceOffsetsBucket.RAMQuotaMB) + if err != nil { + return nil, fmt.Errorf("Creating bucket %s for couchbase server %s: %w", pieceOffsetsBucket, settings.ConnectString, err) + } + db.pieceOffsets = pieceOffsets.DefaultCollection() + + meta, err := CreateBucket(ctx, cluster, metaBucket, 156) + if err != nil { + return nil, fmt.Errorf("Creating bucket %s for couchbase server %s: %w", metaBucket, settings.ConnectString, err) + } + + err = createCollection(ctx, cluster, meta, "piece-tracker") if err != nil { return nil, err } + err = createCollection(ctx, cluster, meta, "piece-flagged") + if err != nil { + return nil, err + } + + return db, nil +} + +func createCollection(ctx context.Context, cluster *gocb.Cluster, bucket *gocb.Bucket, collName string) error { + err := bucket.Collections().CreateCollection(gocb.CollectionSpec{ + Name: collName, + ScopeName: bucket.DefaultScope().Name(), + }, &gocb.CreateCollectionOptions{Context: ctx}) + if err != nil { + if !errors.Is(err, gocb.ErrCollectionExists) { + return fmt.Errorf("Creating %s collection: %w", collName, err) + } + } - return &DB{col: bucket.DefaultCollection()}, nil + time.Sleep(time.Second) + + err = cluster.QueryIndexes().CreatePrimaryIndex(bucket.Name(), &gocb.CreatePrimaryQueryIndexOptions{ + Context: ctx, + ScopeName: bucket.DefaultScope().Name(), + CollectionName: collName, + IgnoreIfExists: true, + }) + if err != nil { + return fmt.Errorf("creating primary index on %s.%s: %w", bucket.Name(), collName, err) + } + + return nil } -// NextCursor -func (db *DB) NextCursor(ctx context.Context) (uint64, string, error) { - b, err := db.Get(ctx, dskeyNextCursor) +func pingCluster(ctx context.Context, cluster *gocb.Cluster, connectString string) error { + res, err := cluster.Ping(&gocb.PingOptions{ + Timeout: connectTimeout, + Context: ctx, + }) + if err == nil { + for svc, png := range res.Services { + if len(png) > 0 && png[0].State != gocb.PingStateOk { + err = fmt.Errorf("connecting to %s service", ServiceName(svc)) + break + } + } + } + if err != nil { - return 0, "", err + msg := fmt.Sprintf("Connecting to couchbase server %s", connectString) + return fmt.Errorf(msg+": %w\nCheck the couchbase server is running and the username / password are correct", err) } - cursor, _ := binary.Uvarint(b) - return cursor, fmt.Sprintf("%d", cursor) + "/", nil // adding "/" because of Query method in go-datastore + return nil } -// SetNextCursor -func (db *DB) SetNextCursor(ctx context.Context, cursor uint64) error { - buf := make([]byte, size) - binary.PutUvarint(buf, cursor) +func CreateBucket(ctx context.Context, cluster *gocb.Cluster, bucketName string, ramMb uint64) (*gocb.Bucket, error) { + _, err := cluster.Buckets().GetBucket(bucketName, &gocb.GetBucketOptions{Context: ctx, Timeout: connectTimeout}) + if err != nil { + if !errors.Is(err, gocb.ErrBucketNotFound) { + msg := fmt.Sprintf("getting bucket %s", bucketName) + return nil, fmt.Errorf(msg+": %w\nCheck the couchbase server is running and the username / password are correct", err) + } + + err = cluster.Buckets().CreateBucket(gocb.CreateBucketSettings{ + BucketSettings: gocb.BucketSettings{ + Name: bucketName, + RAMQuotaMB: ramMb, + BucketType: gocb.CouchbaseBucketType, + // The default eviction policy requires couchbase to keep all + // keys (and metadata) in memory. So use an eviction policy + // that allows keys to be stored on disk (but not in memory). + EvictionPolicy: gocb.EvictionPolicyTypeFull, + }, + }, &gocb.CreateBucketOptions{Context: ctx}) + if err != nil { + return nil, fmt.Errorf("creating bucket %s: %w", bucketName, err) + } + + // TODO: For some reason WaitUntilReady times out if we don't put + // this sleep here + time.Sleep(time.Second) + } - return db.Put(ctx, dskeyNextCursor, buf) + err = cluster.QueryIndexes().CreatePrimaryIndex(bucketName, &gocb.CreatePrimaryQueryIndexOptions{ + IgnoreIfExists: true, + Context: ctx, + }) + if err != nil { + return nil, fmt.Errorf("creating primary index on %s: %w", bucketName, err) + } + + bucket := cluster.Bucket(bucketName) + err = bucket.WaitUntilReady(5*time.Second, nil) + if err != nil { + return nil, fmt.Errorf("waiting for couchbase bucket to be ready: %w", err) + } + + return bucket, nil } // GetPieceCidsByMultihash func (db *DB) GetPieceCidsByMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) { - key := datastore.NewKey(fmt.Sprintf("%s%s", sprefixMhtoPieceCids, mh.String())) + ctx, span := tracing.Tracer.Start(ctx, "db.get_piece_cids_by_multihash") + defer span.End() - val, err := db.Get(ctx, key) - if err != nil { - return nil, fmt.Errorf("failed to get value for multihash %s, err: %w", mh, err) - } + pieceCids, _, err := db.getPieceCidsForMultihash(ctx, mh) + return pieceCids, err +} - var pcids []cid.Cid - if err := json.Unmarshal(val, &pcids); err != nil { - return nil, fmt.Errorf("failed to unmarshal pieceCids slice: %w", err) +const throttleSize = 32 + +// SetMultihashesToPieceCid +func (db *DB) SetMultihashesToPieceCid(ctx context.Context, mhs []multihash.Multihash, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "db.set_multihashes_to_piece_cid") + defer span.End() + + throttle := make(chan struct{}, throttleSize) + var eg errgroup.Group + for _, mh := range mhs { + mh := mh + + throttle <- struct{}{} + eg.Go(func() error { + defer func() { <-throttle }() + + return db.withCasRetry("multihash -> pieces", func() error { + // Insert a tuple into the bucket: multihash -> [piece cid] + cbKey := encodeMultihashAsKey(trimMultihash(mh)) + _, err := db.mhToPieces.Insert(cbKey, pieceCid.Bytes(), &gocb.InsertOptions{ + Context: ctx, + Transcoder: binaryTranscoder, + }) + if err == nil { + return nil + } + + // If the value already exists, it's not an error, we'll just + // add the piece cid to the existing set of piece cids + isDocExists := errors.Is(err, gocb.ErrDocumentExists) + if !isDocExists { + // If there was some other error, return it + return fmt.Errorf("adding mapping multihash %s -> piece %s: insert doc: %w", mh, pieceCid, err) + } + + pieceCids, getRes, err := db.getPieceCidsForMultihash(ctx, mh) + if err != nil { + return fmt.Errorf("adding mapping multihash %s -> piece %s: get existing piece cids: %w", mh, pieceCid, err) + } + + // Check if the array already contains the new piece cid + for _, pcid := range pieceCids { + if pcid == pieceCid { + return nil + } + } + + // Add the new piece cid to the array + pieceCids = append(pieceCids, pieceCid) + err = db.setPieceCidsForMultihash(ctx, mh, pieceCids, getRes.Cas()) + if err != nil { + return fmt.Errorf("adding mapping multihash %s -> piece %s: replace: %w", mh, pieceCid, err) + } + + return nil + }) + }) } - return pcids, nil + return eg.Wait() } -func (db *DB) Get(ctx context.Context, key datastore.Key) (value []byte, err error) { - var getResult *gocb.GetResult - getResult, err = db.col.Get("u:"+key.String(), nil) +func (db *DB) getPieceCidsForMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, *gocb.GetResult, error) { + cbKey := encodeMultihashAsKey(trimMultihash(mh)) + getRes, err := db.mhToPieces.Get(cbKey, &gocb.GetOptions{ + Transcoder: binaryTranscoder, + Context: ctx, + }) if err != nil { - return nil, err + return nil, nil, fmt.Errorf("get piece cids for multihash %s: %w", mh, err) } - //cas := getResult.Cas() + var pieceCidsBytes []byte + err = getRes.Content(&pieceCidsBytes) + if err != nil { + return nil, nil, fmt.Errorf("get content for multihash %s: %w", mh, err) + } - var val []byte - err = getResult.Content(&val) + pieceCids, err := bytesToCids(pieceCidsBytes) if err != nil { - return nil, err + return nil, nil, fmt.Errorf("read piece cid for multihash %s: %w", mh, err) } + return pieceCids, getRes, nil +} - return val, nil +func (db *DB) setPieceCidsForMultihash(ctx context.Context, mh multihash.Multihash, pieceCids []cid.Cid, cas gocb.Cas) error { + bz := cidsToBytes(pieceCids) + cbKey := encodeMultihashAsKey(trimMultihash(mh)) + _, err := db.mhToPieces.Replace(cbKey, bz, &gocb.ReplaceOptions{ + Context: ctx, + Transcoder: binaryTranscoder, + Cas: cas, + }) + if err != nil { + return fmt.Errorf("setting multihash %s -> piece cids: %w", mh, err) + } + + return nil } -func (db *DB) Put(ctx context.Context, key datastore.Key, value []byte) error { - _, err := db.col.Upsert("u:"+key.String(), value, nil) +func (db *DB) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { + ctx, span := tracing.Tracer.Start(ctx, "db.set_car_size") + defer span.End() - return err + return db.mutatePieceMetadata(ctx, pieceCid, "set-car-size", func(metadata CouchbaseMetadata) *CouchbaseMetadata { + // Set the car size on each deal (should be the same for all deals) + var deals []model.DealInfo + for _, dl := range metadata.Deals { + dl.CarLength = size + + deals = append(deals, dl) + } + metadata.Deals = deals + return &metadata + }) } -// SetMultihashToPieceCid -func (db *DB) SetMultihashesToPieceCid(ctx context.Context, recs []carindex.Record, pieceCid cid.Cid) error { - for _, r := range recs { - mh := r.Cid.Hash() +func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr error) error { + ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored") + defer span.End() - err := func() error { - key := datastore.NewKey(fmt.Sprintf("%s%s", sprefixMhtoPieceCids, mh.String())) + return db.mutatePieceMetadata(ctx, pieceCid, "mark-index-errored", func(metadata CouchbaseMetadata) *CouchbaseMetadata { + // If the error was already set, don't overwrite it + if metadata.Error != "" { + // If the error state has already been set, don't over-write the existing error + return nil + } - // do we already have an entry for this multihash ? - val, err := db.Get(ctx, key) - if err != nil && err != ds.ErrNotFound { - return fmt.Errorf("failed to get value for multihash %s, err: %w", mh, err) - } + // Set the error state + metadata.Error = idxErr.Error() + metadata.ErrorType = fmt.Sprintf("%T", idxErr) - // if we don't have an existing entry for this mh, create one - if err == ds.ErrNotFound { - v := []cid.Cid{pieceCid} - b, err := json.Marshal(v) - if err != nil { - return fmt.Errorf("failed to marshal pieceCids slice: %w", err) - } + return &metadata + }) +} - if err := db.Put(ctx, key, b); err != nil { - return fmt.Errorf("failed to put mh=%s, err=%w", mh, err) - } +func (db *DB) MarkIndexingComplete(ctx context.Context, pieceCid cid.Cid, blockCount int, isCompleteIndex bool) error { + ctx, span := tracing.Tracer.Start(ctx, "db.mark_indexing_complete") + defer span.End() + + return db.mutatePieceMetadata(ctx, pieceCid, "mark-indexing-complete", func(metadata CouchbaseMetadata) *CouchbaseMetadata { + // Mark indexing as complete + metadata.IndexedAt = time.Now() + metadata.CompleteIndex = isCompleteIndex + metadata.BlockCount = blockCount + metadata.Error = "" + metadata.ErrorType = "" + if metadata.Deals == nil { + metadata.Deals = []model.DealInfo{} + } + return &metadata + }) +} + +func (db *DB) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { + ctx, span := tracing.Tracer.Start(ctx, "db.add_deal_for_piece") + defer span.End() + + return db.mutatePieceMetadata(ctx, pieceCid, "add-deal-for-piece", func(md CouchbaseMetadata) *CouchbaseMetadata { + // Check if the deal has already been added + for _, dl := range md.Deals { + if dl == dealInfo { return nil } + } - // else, append the pieceCid to the existing list - var pcids []cid.Cid - if err := json.Unmarshal(val, &pcids); err != nil { - return fmt.Errorf("failed to unmarshal pieceCids slice: %w", err) - } + // Add the deal to the list. + // Note: we can't use ArrayAddUniqueSpec here because it only works + // with primitives, not objects. + md.Deals = append(md.Deals, dealInfo) + return &md + }) +} - // if we already have the pieceCid indexed for the multihash, nothing to do here. - if has(pcids, pieceCid) { - return nil +type mutateMetadata func(CouchbaseMetadata) *CouchbaseMetadata + +func (db *DB) mutatePieceMetadata(ctx context.Context, pieceCid cid.Cid, opName string, mutate mutateMetadata) error { + return db.withCasRetry(opName, func() error { + // Get the piece metadata from the db + md := newCouchbaseMetadata() + var pieceMetaExists bool + cbKey := toCouchKey(pieceCid.String()) + getResult, err := db.pcidToMeta.Get(cbKey, &gocb.GetOptions{Context: ctx}) + if err == nil { + pieceMetaExists = true + err = getResult.Content(&md) + if err != nil { + return fmt.Errorf("getting piece cid to metadata content for piece %s: %w", pieceCid, err) } + } else if !isNotFoundErr(err) { + return fmt.Errorf("getting piece cid metadata for piece %s: %w", pieceCid, err) + } + + // Apply the mutation to the metadata + newMetadata := mutate(md) + if newMetadata == nil { + // If there was no mutation applied, just return immediately + return nil + } + + // Write the piece metadata back to the db + if pieceMetaExists { + _, err = db.pcidToMeta.Replace(cbKey, newMetadata, &gocb.ReplaceOptions{ + Context: ctx, + Cas: getResult.Cas(), + }) + } else { + _, err = db.pcidToMeta.Insert(cbKey, newMetadata, &gocb.InsertOptions{Context: ctx}) + } + if err != nil { + return fmt.Errorf("setting piece %s metadata: %w", pieceCid, err) + } - pcids = append(pcids, pieceCid) + return nil + }) +} + +// GetPieceCidToMetadata +func (db *DB) GetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid) (CouchbaseMetadata, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.get_piece_cid_to_metadata") + defer span.End() + + var getResult *gocb.GetResult + k := toCouchKey(pieceCid.String()) + getResult, err := db.pcidToMeta.Get(k, &gocb.GetOptions{Context: ctx}) + if err != nil { + return CouchbaseMetadata{}, fmt.Errorf("getting piece cid to metadata for piece %s: %w", pieceCid, err) + } + + var metadata CouchbaseMetadata + err = getResult.Content(&metadata) + if err != nil { + return CouchbaseMetadata{}, fmt.Errorf("getting piece cid to metadata content for piece %s: %w", pieceCid, err) + } - b, err := json.Marshal(pcids) + return metadata, nil +} + +// GetOffsetSize gets the offset and size of the multihash in the given piece. +// Note that recordCount is needed in order to determine which shard the multihash is in. +func (db *DB) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash multihash.Multihash, recordCount int) (*model.OffsetSize, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.get_offset_size") + defer span.End() + + // Get the prefix for the shard that the multihash is in + shardPrefixBitCount, _ := getShardPrefixBitCount(recordCount) + mask := get2ByteMask(shardPrefixBitCount) + shardPrefix := hashToShardPrefix(hash, mask) + + // Get the map of multihash -> offset/size. + // Note: This doesn't actually fetch the map, it just gets a reference to it. + cbKey := toCouchKey(pieceCid.String() + shardPrefix) + cbMap := db.pieceOffsets.Map(cbKey) + + // Get the offset/size from the map. + // Note: This doesn't actually fetch the whole map, it tells couchbase to + // find the key in the map on the server side, and return the value. + var val string + err := cbMap.At(encodeMultihashAsPath(hash), &val) + if err != nil { + return nil, fmt.Errorf("getting offset/size for piece %s multihash %s: %w", pieceCid, hash, err) + } + + var ofsz model.OffsetSize + err = ofsz.UnmarshallBase64(val) + if err != nil { + return nil, fmt.Errorf("parsing piece %s offset / size value '%s': %w", pieceCid, val, err) + } + return &ofsz, nil +} + +// AllRecords gets all the mulithash -> offset/size mappings in a given piece. +// Note that recordCount is needed in order to determine the shard structure. +func (db *DB) AllRecords(ctx context.Context, pieceCid cid.Cid, recordCount int) ([]model.Record, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.all_records") + defer span.End() + + // Get the number of shards + _, totalShards := getShardPrefixBitCount(recordCount) + + recs := make([]model.Record, 0, recordCount) + recsShard := make([][]model.Record, totalShards) + + var eg errgroup.Group + + span.SetAttributes(attribute.Int("shards", totalShards)) + span.SetAttributes(attribute.Int("recs", recordCount)) + for i := 0; i < totalShards; i++ { + i := i + recsShard[i] = make([]model.Record, 0, recordCount) + eg.Go(func() error { + // Get the map of multihash -> offset/size for the shard + shardPrefix, err := getShardPrefix(i) if err != nil { - return fmt.Errorf("failed to marshal pieceCids slice: %w", err) + return err } - if err := db.Put(ctx, key, b); err != nil { - return fmt.Errorf("failed to put mh=%s, err%w", mh, err) + cbKey := toCouchKey(pieceCid.String() + shardPrefix) + cbMap := db.pieceOffsets.Map(cbKey) + + _, spanIter := tracing.Tracer.Start(ctx, "db.iter") + recMap, err := cbMap.Iterator() + spanIter.End() + if err != nil { + if isNotFoundErr(err) { + // If there are no records in a particular shard just skip the shard + return nil + } + return fmt.Errorf("getting all records for piece %s: %w", pieceCid, err) + } + + span.SetAttributes(attribute.Int(fmt.Sprintf("map_%d", i), len(recMap))) + + _, spanMap := tracing.Tracer.Start(ctx, "db.recMap") + + // Get each value in the map + for mhStr, offsetSizeIfce := range recMap { + mh, err := decodeMultihashAsPath(mhStr) + if err != nil { + return fmt.Errorf("parsing multihash: piece cid %s multihash %s: %w", pieceCid, mhStr, err) + } + + val, ok := offsetSizeIfce.(string) + if !ok { + return fmt.Errorf("unexpected type for piece cid %s offset/size value: %T", pieceCid, offsetSizeIfce) + } + + var ofsz model.OffsetSize + err = ofsz.UnmarshallBase64(val) + if err != nil { + return fmt.Errorf("parsing piece %s offset / size value '%s': %w", pieceCid, val, err) + } + + recsShard[i] = append(recsShard[i], model.Record{Cid: cid.NewCidV1(cid.Raw, mh), OffsetSize: ofsz}) } + spanMap.End() + return nil - }() + }) + } + + err := eg.Wait() + if err != nil { + return nil, err + } + + for i := 0; i < totalShards; i++ { + recs = append(recs, recsShard[i]...) + } + + return recs, nil +} + +// AddIndexRecords +func (db *DB) AddIndexRecords(ctx context.Context, pieceCid cid.Cid, recs []model.Record) error { + ctx, span := tracing.Tracer.Start(ctx, "db.add_index_records") + span.SetAttributes(attribute.Int("recs", len(recs))) + defer span.End() + + if len(recs) > maxRecsPerPiece { + return fmt.Errorf("index for piece %s too large: %d records (size limit is %d)", pieceCid, len(recs), maxRecsPerPiece) + } + + // Get the number of bits in the shard prefix, and the total number of shards + shardPrefixBitCount, totalShards := getShardPrefixBitCount(len(recs)) + + // Initialize the multihash -> offset/size map for each shard + type mhToOffsetSizeMap map[string]string + shardMaps := make(map[string]mhToOffsetSizeMap, totalShards) + for i := 0; i < totalShards; i++ { + shardPrefix, err := getShardPrefix(i) if err != nil { return err } + shardMaps[shardPrefix] = make(map[string]string) + } + + // Create a mask of the required number of bits + // eg 3 bit mask = 0000 0000 0000 0111 + mask := get2ByteMask(shardPrefixBitCount) + + // For each record + for _, rec := range recs { + // Apply the bit mask to the last 2 bytes of the multihash to get the + // shard prefix + hash := rec.Cid.Hash() + shardPrefix := hashToShardPrefix(hash, mask) + + // Add the record to the shard's map + mhenc := encodeMultihashAsPath(hash) + shardMaps[shardPrefix][mhenc] = rec.MarshallBase64() + } + + // Add each shard's map to couchbase + for shardPrefix, shardMap := range shardMaps { + if len(shardMap) == 0 { + continue + } + + cbKey := toCouchKey(pieceCid.String() + shardPrefix) + _, err := db.pieceOffsets.Upsert(cbKey, shardMap, &gocb.UpsertOptions{Context: ctx}) + if err != nil { + return fmt.Errorf("adding offset / sizes for piece %s: %w", pieceCid, err) + } } return nil } -// SetPieceCidToMetadata -func (db *DB) SetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid, md model.Metadata) error { - b, err := json.Marshal(md) +func (db *DB) ListPieces(ctx context.Context) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.list_pieces") + defer span.End() + + res, err := db.query(ctx, "SELECT META().id FROM `"+pieceCidToMetadataBucket+"`") if err != nil { - return err + return nil, fmt.Errorf("getting keys from %s: %w", pieceCidToMetadataBucket, err) } - key := datastore.NewKey(fmt.Sprintf("%s%s", sprefixPieceCidToCursor, pieceCid.String())) + return db.listPieces(res) +} - return db.Put(ctx, key, b) +const piecesToTrackerBatchSize = 1024 +const trackerCheckBatchSize = 1024 + +func (db *DB) NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) { + keepInserting := true + for keepInserting { + // Add any new pieces into the piece status tracking table + qry := "INSERT INTO `" + metaBucket + "`._default.`piece-tracker` (KEY k, VALUE v) " + qry += "SELECT " + qry += " META(pieceMeta).id AS k, " + qry += " {'CreatedAt': NOW_LOCAL(), 'UpdatedAt': null} AS v " + qry += "FROM `" + pieceCidToMetadataBucket + "` AS pieceMeta " + qry += "WHERE META(pieceMeta).id NOT IN (" + qry += " SELECT RAW META(pieceTracker).id FROM `" + metaBucket + "`._default.`piece-tracker` AS pieceTracker" + qry += ") " + qry += fmt.Sprintf("LIMIT %d", piecesToTrackerBatchSize) + + res, err := db.mutate(ctx, qry) + if err != nil { + return nil, fmt.Errorf("executing insert into piece-tracker: %w", err) + } + + // If there were enough remaining rows to fill an entire batch, + // keep inserting a new batch of rows + queryMeta, err := res.MetaData() + if err != nil { + return nil, fmt.Errorf("getting query metadata: %w", err) + } + keepInserting = queryMeta.Metrics.MutationCount == piecesToTrackerBatchSize + } + + // Work out how frequently to check each piece, based on how many pieces + // there are. + // Any pieces that have not been checked in the last pieceCheckPeriod + // will be checked now (eg check all pieces that haven't been checked + // for 10s) + pieceCheckPeriod, err := db.getPieceCheckPeriod(ctx) + if err != nil { + return nil, fmt.Errorf("getting piece check period: %w", err) + } + + // Get all pieces from the piece tracker table that have not been updated + // since the last piece check period. + // Simultaneously set the UpdatedAt field so that these pieces are marked + // as checked (and will not be returned until the next piece check period + // elapses again). + // Note that we limit the number of rows to fetch so as not to overload the + // system. Any rows beyond the limit will be fetched the next time + // NextPiecesToCheck is called. + qry := "UPDATE `" + metaBucket + "`._default.`piece-tracker` as outerPT " + qry += "SET UpdatedAt = NOW_LOCAL() " + qry += "WHERE META(outerPT).id IN (" + qry += " SELECT RAW META(innerPT).id FROM `" + metaBucket + "`._default.`piece-tracker` AS innerPT " + qry += " WHERE " + qry += " innerPT.UpdatedAt IS NULL OR" + qry += " innerPT.UpdatedAt <= DATE_ADD_STR(NOW_LOCAL(), ?, 'millisecond') " + qry += fmt.Sprintf("LIMIT %d", trackerCheckBatchSize) + qry += ") " + qry += "RETURNING META().id" + + res, err := db.query(ctx, qry, -pieceCheckPeriod.Milliseconds()) + if err != nil { + return nil, fmt.Errorf("adding piece meta info to piece tracker table: %w", err) + } + + return db.listPieces(res) } -// GetPieceCidToMetadata -func (db *DB) GetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { - var metadata model.Metadata +// The minimum frequency with which to check pieces for errors (eg bad index) +var MinPieceCheckPeriod = 30 * time.Second - key := datastore.NewKey(fmt.Sprintf("%s%s", sprefixPieceCidToCursor, pieceCid.String())) +// Work out how frequently to check each piece, based on how many pieces +// there are: if there are many pieces, each piece will be checked +// less frequently +func (db *DB) getPieceCheckPeriod(ctx context.Context) (time.Duration, error) { + countRes, err := db.query(ctx, "SELECT COUNT(*) AS c FROM `"+metaBucket+"`._default.`piece-tracker`") + if err != nil { + return 0, fmt.Errorf("reading row from %s: %w", metaBucket, err) + } - b, err := db.Get(ctx, key) + var countRowData map[string]int + err = countRes.One(&countRowData) if err != nil { - return metadata, err + return 0, fmt.Errorf("reading row data from %s: %w", metaBucket, err) + } + count, ok := countRowData["c"] + if !ok { + return 0, fmt.Errorf("unexpected row data reading count: missing count column") } - err = json.Unmarshal(b, &metadata) + // Check period: + // - 1k pieces; every 10s + // - 100k pieces; every 15m + // - 1m pieces; every 2 hours + period := time.Duration(count*10) * time.Millisecond + if period < MinPieceCheckPeriod { + period = MinPieceCheckPeriod + } + + return period, nil +} + +func (db *DB) FlagPiece(ctx context.Context, pieceCid cid.Cid) error { + qry := "INSERT INTO `" + metaBucket + "`._default.`piece-flagged` (KEY, VALUE) " + qry += "VALUES (?, { 'CreatedAt': NOW_LOCAL(), 'UpdatedAt': NOW_LOCAL() })" + _, err := db.mutate(ctx, qry, toCouchKey(pieceCid.String())) if err != nil { - return metadata, err + if !errors.Is(err, gocb.ErrDocumentExists) { + return fmt.Errorf("flagging piece %s: inserting row: %w", pieceCid, err) + } + + qry := "UPDATE `" + metaBucket + "`._default.`piece-flagged` " + qry += "SET UpdatedAt = NOW_LOCAL() " + qry += "WHERE META(id) = ?" + _, err = db.mutate(ctx, qry, toCouchKey(pieceCid.String())) + if err != nil { + return fmt.Errorf("flagging piece %s: updating row: %w", pieceCid, err) + } } + return nil +} - return metadata, nil +func (db *DB) UnflagPiece(ctx context.Context, pieceCid cid.Cid) error { + qry := "DELETE FROM `" + metaBucket + "`._default.`piece-flagged` WHERE META().id = ?" + _, err := db.mutate(ctx, qry, toCouchKey(pieceCid.String())) + if err != nil { + return fmt.Errorf("unflagging piece %s: %w", pieceCid, err) + } + return nil } -// AllRecords -func (db *DB) AllRecords(ctx context.Context, cursor uint64) ([]model.Record, error) { - return nil, errors.New("not impl") - //var records []model.Record +func (db *DB) FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.list_flagged_pieces") + defer span.End() + + args := []interface{}{} + tableName := "`" + metaBucket + "`._default.`piece-flagged`" + qry := "SELECT META(pieceFlagged).id, CreatedAt FROM " + tableName + " AS pieceFlagged " + if cursor != nil { + qry += "WHERE CreatedAt < ? " + args = append(args, cursor) + } + qry += "ORDER BY CreatedAt desc " + + qry += "LIMIT ? OFFSET ?" + args = append(args, limit, offset) - //buf := make([]byte, size) - //binary.PutUvarint(buf, cursor) + res, err := db.query(ctx, qry, args...) + if err != nil { + return nil, fmt.Errorf("getting keys from %s: %w", tableName, err) + } - //var q query.Query - //q.Prefix = fmt.Sprintf("%d/", cursor) - //results, err := db.Query(ctx, q) - //if err != nil { - //return nil, err - //} + var pieces []model.FlaggedPiece + var rowData map[string]string + for res.Next() { + err := res.Row(&rowData) + if err != nil { + return nil, fmt.Errorf("reading row from %s: %w", pieceCidToMetadataBucket, err) + } - //for { - //r, ok := results.NextSync() - //if !ok { - //break - //} + couchKey, ok := rowData["id"] + if !ok { + return nil, fmt.Errorf("unexpected row data %s reading row from %s: missing id", rowData, pieceCidToMetadataBucket) + } - //k := r.Key[len(q.Prefix)+1:] + c, err := cid.Parse(couchKey) + if err != nil { + return nil, fmt.Errorf("parsing piece cid from couchbase key '%s': %w", couchKey, err) + } - //m, err := multihash.FromHexString(k) - //if err != nil { - //return nil, err - //} + createdAtStr, ok := rowData["CreatedAt"] + if !ok { + return nil, fmt.Errorf("unexpected row data %s reading row from %s: missing CreatedAt", rowData, pieceCidToMetadataBucket) + } + createdAt, err := time.Parse(time.RFC3339, createdAtStr) + if err != nil { + return nil, fmt.Errorf("parsing flagged piece CreatedAt from '%s': %w", couchKey, err) + } - //kcid := cid.NewCidV1(cid.Raw, m) + pieces = append(pieces, model.FlaggedPiece{ + CreatedAt: createdAt, + PieceCid: c, + }) + } - //offset, _ := binary.Uvarint(r.Value) + err = res.Err() + if err != nil { + return nil, fmt.Errorf("reading stream from %s: %w", pieceCidToMetadataBucket, err) + } + + return pieces, nil +} + +func (db *DB) FlaggedPiecesCount(ctx context.Context) (int, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.count_flagged_pieces") + defer span.End() - //records = append(records, model.Record{ - //Cid: kcid, - //Offset: offset, - //}) - //} + tableName := "`" + metaBucket + "`._default.`piece-flagged`" + qry := "SELECT COUNT(*) as cnt FROM " + tableName - //return records, nil + res, err := db.query(ctx, qry) + if err != nil { + return 0, fmt.Errorf("getting count of flagged pieces: %w", err) + } + + var m map[string]int + err = res.One(&m) + if err != nil { + return 0, fmt.Errorf("getting count of flagged pieces from result: %w", err) + } + count, ok := m["cnt"] + if !ok { + return 0, fmt.Errorf("missing expected result column in count query") + } + + return count, err } -// AddOffset -func (db *DB) AddOffset(ctx context.Context, cursorPrefix string, m multihash.Multihash, offset uint64) error { - key := datastore.NewKey(fmt.Sprintf("%s%s", cursorPrefix, m.String())) +func (db *DB) listPieces(res *gocb.QueryResult) ([]cid.Cid, error) { + var pieceCids []cid.Cid + var rowData map[string]string + for res.Next() { + err := res.Row(&rowData) + if err != nil { + return nil, fmt.Errorf("reading row: %w", err) + } + + couchKey, ok := rowData["id"] + if !ok { + return nil, fmt.Errorf("unexpected row data %s reading piece list row", rowData) + } + + c, err := cid.Parse(couchKey) + if err != nil { + return nil, fmt.Errorf("parsing piece cid from couchbase key '%s': %w", couchKey, err) + } + + pieceCids = append(pieceCids, c) + } + + err := res.Err() + if err != nil { + return nil, fmt.Errorf("reading stream from piece list data: %w", err) + } - value := make([]byte, size) - binary.PutUvarint(value, offset) + return pieceCids, nil +} - return db.Put(ctx, key, value) +func (db *DB) query(ctx context.Context, qry string, args ...interface{}) (*gocb.QueryResult, error) { + opts := &gocb.QueryOptions{ + Context: ctx, + PositionalParameters: args, + } + if db.settings.TestMode { + // In test mode, require immediate consistency for reads: + // wait for all documents to complete updating before performing + // the query + opts.ScanConsistency = gocb.QueryScanConsistencyRequestPlus + } + return db.cluster.Query(qry, opts) } -// GetOffset -func (db *DB) GetOffset(ctx context.Context, cursorPrefix string, m multihash.Multihash) (uint64, error) { - key := datastore.NewKey(fmt.Sprintf("%s%s", cursorPrefix, m.String())) +func (db *DB) mutate(ctx context.Context, qry string, args ...interface{}) (*gocb.QueryResult, error) { + // Execute the query + opts := &gocb.QueryOptions{ + Context: ctx, + PositionalParameters: args, + } + res, err := db.cluster.Query(qry, opts) + if err != nil { + return nil, fmt.Errorf("executing mutate query: %w", err) + } - b, err := db.Get(ctx, key) + // We have to drain the results in order to close the stream + for res.Next() { + } + err = res.Err() if err != nil { - return 0, err + return nil, fmt.Errorf("draining query stream: %w", err) } - offset, _ := binary.Uvarint(b) - return offset, nil + return res, nil } -func has(list []cid.Cid, v cid.Cid) bool { - for _, l := range list { - if l.Equals(v) { - return true +// Attempt to perform an update operation. If the operation fails due to a +// cas mismatch, or inserting a document at a key that already exists, retry +// several times before giving up. +// Note: cas mismatch is caused when +// - there is a get + update +// - another process applied the update before this process +func (db *DB) withCasRetry(opName string, f func() error) error { + var err error + for i := 0; i < maxCasRetries; i++ { + err = f() + if err == nil { + return nil } + if !errors.Is(err, gocb.ErrCasMismatch) && !errors.Is(err, gocb.ErrDocumentExists) { + return err + } + } + + if err != nil { + log.Warnw("exceeded max compare and swap retries (%d) for "+opName+": %w", maxCasRetries, err) + } + + return err +} + +func toCouchKey(k string) string { + if len(k) > maxCouchKeyLen { + // There is usually important stuff at the beginning and end of a key, + // so cut out the characters in the middle + k = k[:maxCouchKeyLen/2] + k[len(k)-maxCouchKeyLen/2:] } - return false + return k +} + +func isNotFoundErr(err error) bool { + return errors.Is(err, gocb.ErrDocumentNotFound) +} + +func ServiceName(svc gocb.ServiceType) string { + switch svc { + case gocb.ServiceTypeManagement: + return "mgmt" + case gocb.ServiceTypeKeyValue: + return "kv" + case gocb.ServiceTypeViews: + return "views" + case gocb.ServiceTypeQuery: + return "query" + case gocb.ServiceTypeSearch: + return "search" + case gocb.ServiceTypeAnalytics: + return "analytics" + } + return "unknown" +} + +func EndpointStateName(state gocb.EndpointState) string { + switch state { + case gocb.EndpointStateDisconnected: + return "disconnected" + case gocb.EndpointStateConnecting: + return "connecting" + case gocb.EndpointStateConnected: + return "connected" + case gocb.EndpointStateDisconnecting: + return "disconnecting" + } + return "" +} + +// RemoveMetadata +func (db *DB) RemovePieceMetadata(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "db.remove_piece_metadata") + defer span.End() + + return db.withCasRetry("remove-metadata-for-piece", func() error { + var getResult *gocb.GetResult + k := toCouchKey(pieceCid.String()) + getResult, err := db.pcidToMeta.Get(k, &gocb.GetOptions{Context: ctx}) + if err != nil { + if isNotFoundErr(err) { + return nil + } + return fmt.Errorf("getting piece cid to metadata for piece %s: %w", pieceCid, err) + } + + var metadata CouchbaseMetadata + err = getResult.Content(&metadata) + if err != nil { + return fmt.Errorf("getting piece cid to metadata content for piece %s: %w", pieceCid, err) + } + + // Remove all multihashes first, as without Metadata, they cannot be removed. + // This order is important as metadata.BlockCount is required in case RemoveIndexes fails + // and needs to be run manually + if err = db.RemoveIndexes(ctx, pieceCid, metadata.BlockCount); err != nil { + return fmt.Errorf("failed removing index for piece %s: %w", pieceCid, err) + } + + _, err = db.pcidToMeta.Remove(k, &gocb.RemoveOptions{ + Context: ctx, + Cas: getResult.Cas(), + }) + if err != nil { + if isNotFoundErr(err) { + return nil + } + return fmt.Errorf("removing piece %s metadata: %w", pieceCid, err) + } + return nil + }) +} + +// RemoveIndexes +func (db *DB) RemoveIndexes(ctx context.Context, pieceCid cid.Cid, recordCount int) error { + ctx, span := tracing.Tracer.Start(ctx, "db.remove_indexes") + defer span.End() + + // For each shard + _, totalShards := getShardPrefixBitCount(recordCount) + for i := 0; i < totalShards; i++ { + // Get the map of multihash -> offset/size for the shard + shardPrefix, err := getShardPrefix(i) + if err != nil { + return err + } + cbKey := toCouchKey(pieceCid.String() + shardPrefix) + cbMap := db.pieceOffsets.Map(cbKey) + recMap, err := cbMap.Iterator() + if err != nil { + if isNotFoundErr(err) { + // If there are no records in a particular shard just skip the shard + continue + } + return fmt.Errorf("getting all records for piece %s: %w", pieceCid, err) + } + + // For each multihash in the map + for mhStr := range recMap { + mh, err := decodeMultihashAsPath(mhStr) + if err != nil { + return fmt.Errorf("parsing multihash: piece cid %s multihash %s: %w", pieceCid, mhStr, err) + } + + // Remove the piece cid from the mh -> piece cids map + err = db.withCasRetry("remove-piece", func() error { + pieceCids, getRes, err := db.getPieceCidsForMultihash(ctx, mh) + if err != nil { + if isNotFoundErr(err) { + return nil + } + return err + } + + // Remove piece cid from array of piece cids + for i, v := range pieceCids { + if v == pieceCid { + pieceCids[i] = pieceCids[len(pieceCids)-1] + pieceCids = pieceCids[:len(pieceCids)-1] + } + } + + err = db.setPieceCidsForMultihash(ctx, mh, pieceCids, getRes.Cas()) + if err != nil { + return fmt.Errorf("removing piece cid %s from multihash %s -> piece cids: %w", pieceCid, mh, err) + } + + return nil + }) + if err != nil { + return err + } + } + + // Remove the shard + _, err = db.pieceOffsets.Remove(cbKey, &gocb.RemoveOptions{ + Context: ctx, + }) + if err != nil { + if isNotFoundErr(err) { + return nil + } + return err + } + } + return nil +} + +func (db *DB) RemoveDealForPiece(ctx context.Context, dealId string, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "db.remove_deal_for_piece") + defer span.End() + + return db.withCasRetry("remove-deal-for-piece", func() error { + var getResult *gocb.GetResult + k := toCouchKey(pieceCid.String()) + getResult, err := db.pcidToMeta.Get(k, &gocb.GetOptions{Context: ctx}) + if err != nil { + if isNotFoundErr(err) { + return nil + } + return fmt.Errorf("getting piece cid to metadata for piece %s: %w", pieceCid, err) + } + + var metadata CouchbaseMetadata + err = getResult.Content(&metadata) + if err != nil { + return fmt.Errorf("getting piece cid to metadata content for piece %s: %w", pieceCid, err) + } + + for i, v := range metadata.Deals { + if v.DealUuid == dealId { + metadata.Deals[i] = metadata.Deals[len(metadata.Deals)-1] + metadata.Deals = metadata.Deals[:len(metadata.Deals)-1] + break + } + } + // Remove Metadata if removed deal was last one + if len(metadata.Deals) == 0 { + if err := db.RemovePieceMetadata(ctx, pieceCid); err != nil { + fmt.Errorf("Failed to remove the Metadata after removing the last deal: %w", err) + } + return nil + } + + _, err = db.pcidToMeta.Replace(k, metadata, &gocb.ReplaceOptions{ + Context: ctx, + Cas: getResult.Cas(), + }) + if err != nil { + return fmt.Errorf("setting piece %s metadata: %w", pieceCid, err) + } + + return nil + }) } diff --git a/extern/boostd-data/couchbase/encode.go b/extern/boostd-data/couchbase/encode.go new file mode 100644 index 000000000..4726a339f --- /dev/null +++ b/extern/boostd-data/couchbase/encode.go @@ -0,0 +1,80 @@ +package couchbase + +import ( + "encoding/ascii85" + "encoding/base64" + "fmt" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" +) + +func bytesToCids(bz []byte) ([]cid.Cid, error) { + var bytesIdx int + var pieceCids []cid.Cid + for bytesIdx < len(bz) { + readCount, pcid, err := cid.CidFromBytes(bz[bytesIdx:]) + if err != nil { + return nil, fmt.Errorf("parsing bytes to cid: %w", err) + } + + bytesIdx += readCount + pieceCids = append(pieceCids, pcid) + } + return pieceCids, nil +} + +func cidsToBytes(cids []cid.Cid) []byte { + var bz []byte + for _, c := range cids { + bz = append(bz, c.Bytes()...) + } + return bz +} + +// Probability of a collision in two 24 byte hashes (birthday problem): +// 2^(24*8/2) = 8 x 10^28 +const multihashLimitBytes = 24 + +// trimMultihash trims the multihash to the last multihashLimitBytes bytes +func trimMultihash(mh multihash.Multihash) []byte { + var idx int + if len(mh) > multihashLimitBytes { + idx = len(mh) - multihashLimitBytes + } + return mh[idx:] +} + +// encodeMultihashAsKey encodes the multihash as a utf8-conformant string so that it +// can be used as a couchbase key: +// https://docs.couchbase.com/server/current/learn/data/data.html#keys +func encodeMultihashAsKey(mh []byte) string { + // ascii85 encodes to 25% larger than the original bytes and doesn't include + // the space character (which is forbidden by couchbase key syntax) + dest := make([]byte, ascii85.MaxEncodedLen(len(mh))) + n := ascii85.Encode(dest, mh) + return string(dest[:n]) +} + +// encodeMultihashAsPath encodes the multihash as a string that can be used +// as a couchbase path (so that it can be used as a map key): +// https://docs.couchbase.com/go-sdk/current/howtos/subdocument-operations.html#path-syntax +func encodeMultihashAsPath(mh []byte) string { + // base64 encodes to 33% larger than the original bytes and doesn't include + // any characters forbidden by couchbase path syntax + return base64.RawStdEncoding.EncodeToString(mh) +} + +// decodeMultihashAsPath decodes an encoded path string into a multihash +func decodeMultihashAsPath(mhenc string) (multihash.Multihash, error) { + mhbz, err := base64.RawStdEncoding.DecodeString(mhenc) + if err != nil { + return nil, fmt.Errorf("decoding multihash as path: %w", err) + } + + _, mh, err := multihash.MHFromBytes(mhbz) + if err != nil { + return nil, fmt.Errorf("parsing multihash as path from bytes: %w", err) + } + + return mh, nil +} diff --git a/extern/boostd-data/couchbase/encode_test.go b/extern/boostd-data/couchbase/encode_test.go new file mode 100644 index 000000000..abe518574 --- /dev/null +++ b/extern/boostd-data/couchbase/encode_test.go @@ -0,0 +1,79 @@ +package couchbase + +import ( + "fmt" + "github.com/filecoin-project/boost/testutil" + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" + "strings" + "testing" + "unicode/utf8" +) + +func TestCidsToBytes(t *testing.T) { + c1 := testutil.GenerateCid() + c2 := testutil.GenerateCid() + + bz := cidsToBytes([]cid.Cid{c1}) + res, err := bytesToCids(bz) + require.NoError(t, err) + require.Equal(t, c1, res[0]) + + bz = cidsToBytes([]cid.Cid{c1, c2}) + res, err = bytesToCids(bz) + require.NoError(t, err) + require.Equal(t, c1, res[0]) + require.Equal(t, c2, res[1]) +} + +func TestEncodeMultihashAsPath(t *testing.T) { + for i := 0; i < 1000; i++ { + c := testutil.GenerateCid() + mhstr := encodeMultihashAsPath(c.Hash()) + mh, err := decodeMultihashAsPath(mhstr) + if err != nil { + fmt.Printf("%d: %s", i, mhstr) + } + require.NoError(t, err) + require.Equal(t, c.Hash(), mh) + } +} + +// Verifies that encoded paths meet the couchbase requirements: +// https://docs.couchbase.com/go-sdk/current/howtos/subdocument-operations.html#path-syntax +func TestEncodeMultihashAsPathCouchbaseFriendly(t *testing.T) { + for i := 0; i < 1000; i++ { + c := testutil.GenerateCid() + mhstr := encodeMultihashAsPath(c.Hash()) + + // Couchbase paths must be utf8 conformant + isValid := utf8.Valid([]byte(mhstr)) + require.True(t, isValid) + + // Couchbase keys must not contain these characters: space . \ ` [ ] + for _, c := range " .`[]\\" { + has := strings.Contains(mhstr, fmt.Sprintf("%c", c)) + require.False(t, has, "encoded string %s is invalid: contains %c", mhstr, c) + } + } +} + +// Verifies that encoded keys meet the couchbase requirements: +// https://docs.couchbase.com/server/current/learn/data/data.html#keys +func TestEncodeMultihashAsKeyCouchbaseFriendly(t *testing.T) { + for i := 0; i < 1000; i++ { + c := testutil.GenerateCid() + mhstr := encodeMultihashAsKey(c.Hash()) + + // Couchbase keys must be utf8 conformant + isValid := utf8.Valid([]byte(mhstr)) + if !isValid { + fmt.Println("not valid") + } + require.True(t, isValid) + + // Couchbase keys must not contain spaces + hasSpaces := strings.Contains(mhstr, " ") + require.False(t, hasSpaces) + } +} diff --git a/extern/boostd-data/couchbase/service.go b/extern/boostd-data/couchbase/service.go index 2699ec414..b4bd2bb78 100644 --- a/extern/boostd-data/couchbase/service.go +++ b/extern/boostd-data/couchbase/service.go @@ -4,227 +4,399 @@ import ( "context" "errors" "fmt" - "sync" "time" - "github.com/filecoin-project/boost/cmd/boostd-data/model" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/tracing" + "github.com/filecoin-project/boostd-data/svc/types" "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" - "github.com/ipld/go-car/v2/index" - carindex "github.com/ipld/go-car/v2/index" - "github.com/multiformats/go-multicodec" - "github.com/multiformats/go-multihash" mh "github.com/multiformats/go-multihash" + "go.opentelemetry.io/otel/attribute" ) var log = logging.Logger("boostd-data-cb") +type CouchbaseMetadata struct { + model.Metadata + BlockCount int `json:"b"` +} + type Store struct { - sync.Mutex - db *DB + settings DBSettings + db *DB +} + +var _ types.ServiceImpl = (*Store)(nil) + +func NewStore(settings DBSettings) *Store { + return &Store{settings: settings} } -func NewStore() *Store { - db, err := newDB() +func (s *Store) Start(ctx context.Context) error { + db, err := newDB(ctx, s.settings) if err != nil { - panic(err) + return fmt.Errorf("starting couchbase service: %w", err) } - return &Store{ - db: db, - } + s.db = db + return nil } -func (s *Store) AddDealForPiece(pieceCid cid.Cid, dealInfo model.DealInfo) error { +func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { log.Debugw("handle.add-deal-for-piece", "piece-cid", pieceCid) + ctx, span := tracing.Tracer.Start(context.Background(), "store.add_deal_for_piece") + defer span.End() + defer func(now time.Time) { log.Debugw("handled.add-deal-for-piece", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) - return nil + return s.db.AddDealForPiece(ctx, pieceCid, dealInfo) } -func (s *Store) GetOffset(pieceCid cid.Cid, hash mh.Multihash) (uint64, error) { - log.Debugw("handle.get-offset", "piece-cid", pieceCid) +func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { + log.Debugw("handle.set-car-size", "piece-cid", pieceCid, "size", size) + + ctx, span := tracing.Tracer.Start(context.Background(), "store.set-car-size") + defer span.End() defer func(now time.Time) { - log.Debugw("handled.get-offset", "took", fmt.Sprintf("%s", time.Since(now))) + log.Debugw("handled.set-car-size", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) - return 0, nil + err := s.db.SetCarSize(ctx, pieceCid, size) + return normalizePieceCidError(pieceCid, err) } -func (s *Store) GetPieceDeals(pieceCid cid.Cid) ([]model.DealInfo, error) { - log.Debugw("handle.get-piece-deals", "piece-cid", pieceCid) +func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { + log.Debugw("handle.mark-piece-index-errored", "piece-cid", pieceCid, "err", idxErr) + + ctx, span := tracing.Tracer.Start(ctx, "store.mark-piece-index-errored") + defer span.End() defer func(now time.Time) { - log.Debugw("handled.get-piece-deals", "took", fmt.Sprintf("%s", time.Since(now))) + log.Debugw("handled.mark-piece-index-errored", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) - return nil, nil + err := s.db.MarkIndexErrored(ctx, pieceCid, errors.New(idxErr)) + if err != nil { + return normalizePieceCidError(pieceCid, err) + } + + return s.FlagPiece(ctx, pieceCid) } -// Get all pieces that contain a multihash (used when retrieving by payload CID) -func (s *Store) PiecesContainingMultihash(m mh.Multihash) ([]cid.Cid, error) { - log.Debugw("handle.pieces-containing-mh", "mh", m) +func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { + log.Debugw("handle.get-offset-size", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.get_offset_size") + defer span.End() defer func(now time.Time) { - log.Debugw("handled.pieces-containing-mh", "took", fmt.Sprintf("%s", time.Since(now))) + log.Debugw("handled.get-offset-size", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) - return nil, nil + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + err = normalizePieceCidError(pieceCid, err) + return nil, fmt.Errorf("getting piece metadata for piece %s: %w", pieceCid, err) + } + + ofsz, err := s.db.GetOffsetSize(ctx, pieceCid, hash, md.BlockCount) + return ofsz, normalizePieceCidError(pieceCid, err) } -func (s *Store) GetRecords(pieceCid cid.Cid) ([]model.Record, error) { - log.Debugw("handle.get-iterable-index", "piece-cid", pieceCid) +func (s *Store) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { + log.Debugw("handle.get-piece-metadata", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.get_piece_metadata") + defer span.End() defer func(now time.Time) { - log.Debugw("handled.get-iterable-index", "took", fmt.Sprintf("%s", time.Since(now))) + log.Debugw("handled.get-piece-metadata", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) - s.Lock() - defer s.Unlock() - - ctx := context.Background() - md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) if err != nil { - return nil, err + err = normalizePieceCidError(pieceCid, err) + return model.Metadata{}, fmt.Errorf("getting piece metadata for piece %s: %w", pieceCid, err) } - records, err := s.db.AllRecords(ctx, md.Cursor) + return md.Metadata, nil +} + +func (s *Store) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) { + log.Debugw("handle.get-piece-deals", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.get_piece_deals") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.get-piece-deals", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) + + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) if err != nil { - return nil, err + err = normalizePieceCidError(pieceCid, err) + return nil, fmt.Errorf("getting piece deals for piece %s: %w", pieceCid, err) } - return records, nil + return md.Deals, nil } -func (s *Store) GetIndex(pieceCid cid.Cid) ([]model.Record, error) { - log.Warnw("handle.get-index", "pieceCid", pieceCid) +// Get all pieces that contain a multihash (used when retrieving by payload CID) +func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ([]cid.Cid, error) { + log.Debugw("handle.pieces-containing-mh", "mh", m) + + ctx, span := tracing.Tracer.Start(ctx, "store.pieces_containing_multihash") + defer span.End() defer func(now time.Time) { - log.Warnw("handled.get-index", "took", fmt.Sprintf("%s", time.Since(now))) + log.Debugw("handled.pieces-containing-mh", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) - s.Lock() - defer s.Unlock() + pcids, err := s.db.GetPieceCidsByMultihash(ctx, m) + return pcids, normalizeMultihashError(m, err) +} + +func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, error) { + log.Debugw("handle.get-index", "pieceCid", pieceCid) - ctx := context.Background() + ctx, span := tracing.Tracer.Start(ctx, "store.get_index") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.get-index", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) if err != nil { - return nil, err + err = normalizePieceCidError(pieceCid, err) + return nil, fmt.Errorf("getting piece cid %s metadata: %w", pieceCid, err) } - records, err := s.db.AllRecords(ctx, md.Cursor) + records, err := s.db.AllRecords(ctx, pieceCid, md.BlockCount) if err != nil { - return nil, err + err = normalizePieceCidError(pieceCid, err) + return nil, fmt.Errorf("getting all records for piece %s: %w", pieceCid, err) } - log.Warnw("handle.get-index.records", "len(records)", len(records)) + log.Debugw("handle.get-index.records", "len(records)", len(records)) return records, nil } -func (s *Store) AddIndex(pieceCid cid.Cid, records []model.Record) error { - log.Debugw("handle.add-index", "records", len(records)) +func (s *Store) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) { + t, err := s.IndexedAt(ctx, pieceCid) + if err != nil { + return false, err + } + return !t.IsZero(), nil +} + +func (s *Store) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, error) { + log.Debugw("handle.is-complete-index", "pieceCid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.is_incomplete_index") + defer span.End() defer func(now time.Time) { - log.Debugw("handled.add-index", "took", fmt.Sprintf("%s", time.Since(now))) + log.Debugw("handled.is-complete-index", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) - s.Lock() - defer s.Unlock() + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + return false, normalizePieceCidError(pieceCid, err) + } + + return md.CompleteIndex, nil +} + +func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error { + log.Debugw("handle.add-index", "records", len(records)) + + ctx, span := tracing.Tracer.Start(ctx, "store.add_index") + defer span.End() - ctx := context.Background() + start := time.Now() + defer func() { log.Debugw("handled.add-index", "took", time.Since(start).String()) }() - var recs []carindex.Record + // Add a mapping from multihash -> piece cid so that clients can look up + // which pieces contain a multihash + mhs := make([]mh.Multihash, 0, len(records)) for _, r := range records { - recs = append(recs, carindex.Record{ - Cid: r.Cid, - Offset: r.Offset, - }) + mhs = append(mhs, r.Cid.Hash()) } - err := s.db.SetMultihashesToPieceCid(ctx, recs, pieceCid) + setMhStart := time.Now() + err := s.db.SetMultihashesToPieceCid(ctx, mhs, pieceCid) if err != nil { return fmt.Errorf("failed to add entry from mh to pieceCid: %w", err) } + log.Debugw("handled.add-index SetMultihashesToPieceCid", "took", time.Since(setMhStart).String()) - // get and set next cursor (handle synchronization, maybe with CAS) - cursor, keyCursorPrefix, err := s.db.NextCursor(ctx) - if err != nil { - return fmt.Errorf("couldnt generate next cursor: %w", err) - } - - // alloacte metadata for pieceCid - err = s.db.SetNextCursor(ctx, cursor+1) - if err != nil { + // Add a mapping from piece cid -> offset / size of each block so that + // clients can get the block info for all blocks in a piece + addOffsetsStart := time.Now() + if err := s.db.AddIndexRecords(ctx, pieceCid, records); err != nil { return err } + log.Debugw("handled.add-index AddIndexRecords", "took", time.Since(addOffsetsStart).String()) - mis := make(index.MultihashIndexSorted) - err = mis.Load(recs) - if err != nil { - return err - } + return s.db.MarkIndexingComplete(ctx, pieceCid, len(records), isCompleteIndex) +} - var subject index.Index - subject = &mis +func (s *Store) IndexedAt(ctx context.Context, pieceCid cid.Cid) (time.Time, error) { + log.Debugw("handle.indexed-at", "pieceCid", pieceCid) - // process index and store entries - switch idx := subject.(type) { - case index.IterableIndex: - err := idx.ForEach(func(m multihash.Multihash, offset uint64) error { + ctx, span := tracing.Tracer.Start(ctx, "store.indexed_at") + defer span.End() - return s.db.AddOffset(ctx, keyCursorPrefix, m, offset) - }) - if err != nil { - return err - } + defer func(now time.Time) { + log.Debugw("handled.indexed-at", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) - default: - return errors.New(fmt.Sprintf("wanted %v but got %v\n", multicodec.CarMultihashIndexSorted, idx.Codec())) + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil && !isNotFoundErr(err) { + return time.Time{}, err } - // mark that indexing is complete - md := model.Metadata{ - Cursor: cursor, - IndexedAt: time.Now(), + return md.IndexedAt, nil +} + +func (s *Store) ListPieces(ctx context.Context) ([]cid.Cid, error) { + log.Debugw("handle.list-pieces") + + ctx, span := tracing.Tracer.Start(ctx, "store.list_pieces") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.list-pieces", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) + + return s.db.ListPieces(ctx) +} + +func (s *Store) NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.next_pieces_to_check") + defer span.End() + + return s.db.NextPiecesToCheck(ctx) +} + +func (s *Store) FlagPiece(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.flag_piece") + span.SetAttributes(attribute.String("pieceCid", pieceCid.String())) + defer span.End() + + return s.db.FlagPiece(ctx, pieceCid) +} + +func (s *Store) UnflagPiece(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "store.unflag_piece") + span.SetAttributes(attribute.String("pieceCid", pieceCid.String())) + defer span.End() + + return s.db.UnflagPiece(ctx, pieceCid) +} + +func (s *Store) FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces") + var spanCursor int + if cursor != nil { + spanCursor = int(cursor.UnixMilli()) } + span.SetAttributes(attribute.Int("cursor", spanCursor)) + span.SetAttributes(attribute.Int("offset", offset)) + span.SetAttributes(attribute.Int("limit", limit)) + defer span.End() - err = s.db.SetPieceCidToMetadata(ctx, pieceCid, md) - if err != nil { + return s.db.FlaggedPiecesList(ctx, cursor, offset, limit) +} + +func (s *Store) FlaggedPiecesCount(ctx context.Context) (int, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces_count") + defer span.End() + + return s.db.FlaggedPiecesCount(ctx) +} + +// RemoveDealForPiece removes Single deal for pieceCID. If []Deals is empty then Metadata is removed as well +func (s *Store) RemoveDealForPiece(ctx context.Context, pieceCid cid.Cid, dealId string) error { + log.Debugw("handle.remove-deal-for-piece", "piece-cid", pieceCid, "deal-uuid", dealId) + + ctx, span := tracing.Tracer.Start(ctx, "store.remove_deal_for_piece") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.remove-deal-for-piece", "took", time.Since(now).String()) + }(time.Now()) + + return s.db.RemoveDealForPiece(ctx, dealId, pieceCid) +} + +// RemovePieceMetadata removes all Metadata for pieceCID. To be used manually in case of failure +// in RemoveDealForPiece +func (s *Store) RemovePieceMetadata(ctx context.Context, pieceCid cid.Cid) error { + log.Debugw("handle.remove-piece-metadata", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.remove_piece_metadata") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.remove-piece-metadata", "took", time.Since(now).String()) + }(time.Now()) + + if err := s.db.RemovePieceMetadata(ctx, pieceCid); err != nil { return err } - //err = s.db.Sync(ctx, datastore.NewKey(fmt.Sprintf("%d", cursor))) - //if err != nil { - //return err - //} - return nil } -func (s *Store) IndexedAt(pieceCid cid.Cid) (time.Time, error) { - log.Debugw("handle.indexed-at", "pieceCid", pieceCid) +// RemoveIndexes removes all MultiHashes for pieceCID. To be used manually in case of failure +// in RemoveDealForPiece or RemovePieceMetadata. Metadata for the piece must be +// present in the database +func (s *Store) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { + log.Debugw("handle.remove-indexes", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.remove_indexes") + defer span.End() defer func(now time.Time) { - log.Debugw("handled.indexed-at", "took", fmt.Sprintf("%s", time.Since(now))) + log.Debugw("handle.remove-indexes", "took", time.Since(now).String()) }(time.Now()) - s.Lock() - defer s.Unlock() + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + return err + } - ctx := context.Background() + if err := s.db.RemoveIndexes(ctx, pieceCid, md.BlockCount); err != nil { + return err + } - md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) - if err != nil && err != ds.ErrNotFound { - return time.Time{}, err + return nil +} + +func normalizePieceCidError(pieceCid cid.Cid, err error) error { + if err == nil { + return nil + } + if isNotFoundErr(err) { + return fmt.Errorf("piece %s: %s", pieceCid, types.ErrNotFound) } + return err +} - return md.IndexedAt, nil +func normalizeMultihashError(m mh.Multihash, err error) error { + if err == nil { + return nil + } + if isNotFoundErr(err) { + return fmt.Errorf("multihash %s: %s", m, types.ErrNotFound) + } + return err } diff --git a/extern/boostd-data/couchbase/shard.go b/extern/boostd-data/couchbase/shard.go new file mode 100644 index 000000000..b9c36a09c --- /dev/null +++ b/extern/boostd-data/couchbase/shard.go @@ -0,0 +1,64 @@ +package couchbase + +import ( + "fmt" + "github.com/multiformats/go-multihash" +) + +// Couchbase has an upper limit on the size of a value: 20mb +// A JSON-encoded map with 128k keys results in a value of about 8mb in size +// so this is well under the 20mb limit. +// See TestJSONMarshalledMapSize +var maxRecsPerShard = 128 * 1024 + +// Limit the number of shards to 2048. This means there is an upper limit of +// ~270 million blocks per piece, which should be more than enough: +// 64 Gib piece / (2048 * 128 * 1024) = 238 bytes per block +const maxShardsPerPiece = 2048 + +var maxRecsPerPiece = maxShardsPerPiece * maxRecsPerShard + +func getShardPrefixBitCount(recordCount int) (int, int) { + // The number of shards required to store all the keys + requiredShards := (recordCount / maxRecsPerShard) + 1 + // The number of shards that will be created must be a power of 2, + // so get the first power of two that's >= requiredShards + shardPrefixBits := 0 + totalShards := 1 + for totalShards < requiredShards { + shardPrefixBits += 1 + totalShards *= 2 + } + + return shardPrefixBits, totalShards +} + +func getShardPrefix(shardIndex int) (string, error) { + if shardIndex >= 1<<16 { + return "", fmt.Errorf("shard index of size %d does not fit into 2 byte prefix", shardIndex) + } + + shardPrefix := []byte{0, 0} + shardPrefix[1] = byte(shardIndex) + shardPrefix[0] = byte(shardIndex >> 8) + return string(shardPrefix), nil +} + +// Create a 2 byte mask of the required number of bits +// eg 3 bit mask = 0000 0000 0000 0111 +func get2ByteMask(bits int) [2]byte { + buf := [2]byte{0, 0} + buf[1] = (1 << bits) - 1 + if bits >= 8 { + buf[0] = (1 << (bits - 8)) - 1 + } + return buf +} + +// Apply a mask to the last two bytes of the hash to use as the shard prefix +func hashToShardPrefix(hash multihash.Multihash, mask [2]byte) string { + return string([]byte{ + hash[len(hash)-2] & mask[0], + hash[len(hash)-1] & mask[1], + }) +} diff --git a/extern/boostd-data/couchbase/shard_test.go b/extern/boostd-data/couchbase/shard_test.go new file mode 100644 index 000000000..1e409cf11 --- /dev/null +++ b/extern/boostd-data/couchbase/shard_test.go @@ -0,0 +1,218 @@ +package couchbase + +import ( + "context" + "encoding/json" + "fmt" + "github.com/filecoin-project/boost/testutil" + "github.com/filecoin-project/boostd-data/model" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/index" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" + "os" + "testing" + "time" + "unicode/utf8" +) + +func TestBitMask(t *testing.T) { + tcs := []struct { + bits int + expected [2]byte + }{{ + bits: 0, + expected: [2]byte{0, 0}, + }, { + bits: 1, + expected: [2]byte{0, 1}, + }, { + bits: 3, + expected: [2]byte{0, 7}, + }, { + bits: 8, + expected: [2]byte{0, 255}, + }, { + bits: 9, + expected: [2]byte{1, 255}, + }, { + bits: 15, + expected: [2]byte{127, 255}, + }, { + bits: 16, + expected: [2]byte{255, 255}, + }} + + for _, tc := range tcs { + t.Run(fmt.Sprintf("%d", tc.bits), func(t *testing.T) { + mask := get2ByteMask(tc.bits) + require.Equal(t, tc.expected, mask) + }) + } +} + +func TestGetShardPrefix(t *testing.T) { + tcs := []struct { + shardIndex int + expected string + }{{ + shardIndex: 0, + expected: string([]byte{0, 0}), + }, { + shardIndex: 1, + expected: string([]byte{0, 1}), + }, { + shardIndex: 3, + expected: string([]byte{0, 3}), + }, { + shardIndex: 8, + expected: string([]byte{0, 8}), + }, { + shardIndex: 256, + expected: string([]byte{1, 0}), + }, { + shardIndex: 257, + expected: string([]byte{1, 1}), + }, { + shardIndex: (1 << 16) - 1, + expected: string([]byte{255, 255}), + }} + + for _, tc := range tcs { + t.Run(fmt.Sprintf("%d", tc.shardIndex), func(t *testing.T) { + prefix, err := getShardPrefix(tc.shardIndex) + require.NoError(t, err) + require.Equal(t, tc.expected, prefix) + }) + } + + t.Run(fmt.Sprintf("%d", 1<<16), func(t *testing.T) { + _, err := getShardPrefix(1 << 16) + require.Error(t, err) + }) +} + +// Couchbase keys must be valid utf8, so ensure that all shard prefixes are +// valid utf8 +func TestShardPrefixValidUtf8(t *testing.T) { + for i := 0; i < (1<<16)-1; i++ { + prefix, err := getShardPrefix(i) + require.NoError(t, err) + utf8.Valid([]byte(prefix)) + } +} + +var testCouchSettings = DBSettings{ + ConnectString: "couchbase://127.0.0.1", + Auth: DBSettingsAuth{ + Username: "Administrator", + Password: "boostdemo", + }, + PieceMetadataBucket: DBSettingsBucket{ + RAMQuotaMB: 128, + }, + MultihashToPiecesBucket: DBSettingsBucket{ + RAMQuotaMB: 128, + }, + PieceOffsetsBucket: DBSettingsBucket{ + RAMQuotaMB: 128, + }, + TestMode: true, +} + +func TestSharding(t *testing.T) { + // Skip until the tests are refactored such that we can create a couchbase + // instance from docker + t.Skip() + + // Reduce the maximum records per shard such that lots of shards will be created + saved := maxRecsPerShard + maxRecsPerShard = 5 + defer func() { + maxRecsPerShard = saved + }() + + fileSize := 2 * 1024 * 1024 + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Create a new couchbase db + db, err := newDB(context.Background(), testCouchSettings) + require.NoError(t, err) + + // Create a CAR file + randomFilePath, err := testutil.CreateRandomFile(t.TempDir(), 1, fileSize) + require.NoError(t, err) + _, carFilePath, err := testutil.CreateDenseCARv2(t.TempDir(), randomFilePath) + require.NoError(t, err) + carFile, err := os.Open(carFilePath) + require.NoError(t, err) + defer carFile.Close() + idx, err := car.ReadOrGenerateIndex(carFile) + require.NoError(t, err) + + // Get the records from the CAR file + pieceCid, err := cid.Parse("baga6ea4seaqnfhocd544oidrgsss2ahoaomvxuaqxfmlsizljtzsuivjl5hamka") + require.NoError(t, err) + + var recs []model.Record + err = idx.(index.IterableIndex).ForEach(func(m multihash.Multihash, offset uint64) error { + c := cid.NewCidV1(cid.Raw, m) + + recs = append(recs, model.Record{ + Cid: c, + OffsetSize: model.OffsetSize{ + Offset: offset, + Size: 0, + }, + }) + + return nil + }) + require.NoError(t, err) + + // Add the records to the db + err = db.AddIndexRecords(ctx, pieceCid, recs) + require.NoError(t, err) + + // Pick a random record and get its offset and size + rec := recs[len(recs)/2] + hash := rec.Cid.Hash() + offsetSize, err := db.GetOffsetSize(ctx, pieceCid, hash, len(recs)) + require.NoError(t, err) + require.Equal(t, rec.Offset, offsetSize.Offset) + require.Equal(t, rec.Size, offsetSize.Size) + + // Get all records and ensure that they are the same as the records that + // were put in + allRecs, err := db.AllRecords(ctx, pieceCid, len(recs)) + require.NoError(t, err) + require.Equal(t, len(recs), len(allRecs)) + require.ElementsMatch(t, recs, allRecs) +} + +// Verify that the maximum size of a json-marshalled map +// multihash -> offset / size +// is less than the maximum couchbase value size of 20mb +func TestJSONMarshalledMapSize(t *testing.T) { + m := make(map[string]string) + for i := 0; i < maxRecsPerShard; i++ { + c := testutil.GenerateCid() + mhstr := encodeMultihashAsPath(c.Hash()) + offsetSize := model.OffsetSize{ + // Maximum offset and size are 64 Gib + Offset: 64 * 1024 * 1024 * 1024, + Size: 64 * 1024 * 1024 * 1024, + } + m[mhstr] = offsetSize.MarshallBase64() + } + + bz, err := json.Marshal(m) + require.NoError(t, err) + + t.Logf("json-marshalled size: %d", len(bz)) + couchbaseMaxValueSize := 20 * 1024 * 1024 + require.Less(t, len(bz), couchbaseMaxValueSize) +} diff --git a/extern/boostd-data/go.mod b/extern/boostd-data/go.mod index 7859022c3..9d1c890d0 100644 --- a/extern/boostd-data/go.mod +++ b/extern/boostd-data/go.mod @@ -1,94 +1,127 @@ -module github.com/filecoin-project/boost/cmd/boostd-data +module github.com/filecoin-project/boostd-data go 1.18 require ( github.com/couchbase/gocb/v2 v2.5.0 + github.com/davecgh/go-spew v1.1.1 github.com/docker/docker v20.10.7+incompatible github.com/docker/go-connections v0.4.0 - github.com/ethereum/go-ethereum v1.10.19 - github.com/filecoin-project/go-state-types v0.10.0-rc3 + github.com/filecoin-project/boost v1.4.0 + github.com/filecoin-project/go-address v1.0.0 + github.com/filecoin-project/go-jsonrpc v0.1.8 + github.com/filecoin-project/go-state-types v0.1.10 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 - github.com/ipfs/go-cid v0.2.0 - github.com/ipfs/go-datastore v0.5.1 + github.com/ipfs/go-cid v0.3.2 + github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipld/go-car/v2 v2.1.2-0.20220124154420-9c7956a6eb9d - github.com/multiformats/go-multicodec v0.4.1 - github.com/multiformats/go-multihash v0.1.0 + github.com/ipld/go-car/v2 v2.4.2-0.20220707083113-89de8134e58e + github.com/mitchellh/go-homedir v1.1.0 + github.com/multiformats/go-multicodec v0.6.0 + github.com/multiformats/go-multihash v0.2.1 + github.com/stretchr/testify v1.8.1 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/urfave/cli/v2 v2.24.4 go.opentelemetry.io/otel v1.13.0 go.opentelemetry.io/otel/exporters/jaeger v1.13.0 go.opentelemetry.io/otel/sdk v1.13.0 go.opentelemetry.io/otel/trace v1.13.0 - golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2 + golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5 + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 ) require ( - github.com/Microsoft/go-winio v0.5.1 // indirect - github.com/StackExchange/wmi v1.2.1 // indirect - github.com/benbjohnson/clock v1.3.0 // indirect - github.com/containerd/containerd v1.6.6 // indirect + github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/btcsuite/btcd v0.22.1 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect + github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect + github.com/containerd/containerd v1.6.10 // indirect github.com/couchbase/gocbcore/v10 v10.1.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/deckarep/golang-set v1.8.0 // indirect + github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect - github.com/docker/go-units v0.4.0 // indirect - github.com/filecoin-project/go-address v1.1.0 // indirect - github.com/filecoin-project/go-crypto v0.0.1 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-ole/go-ole v1.2.5 // indirect - github.com/go-stack/stack v1.8.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect + github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-block-format v0.0.3 // indirect + github.com/ipfs/go-blockservice v0.3.0 // indirect + github.com/ipfs/go-cidutil v0.1.0 // indirect + github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect + github.com/ipfs/go-ipfs-blocksutil v0.0.1 // indirect + github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect + github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect + github.com/ipfs/go-ipfs-exchange-interface v0.1.0 // indirect + github.com/ipfs/go-ipfs-exchange-offline v0.2.0 // indirect + github.com/ipfs/go-ipfs-files v0.1.1 // indirect + github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect - github.com/ipfs/go-ipld-format v0.2.0 // indirect + github.com/ipfs/go-ipld-format v0.4.0 // indirect github.com/ipfs/go-ipld-legacy v0.1.1 // indirect + github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipfs/go-merkledag v0.6.0 // indirect + github.com/ipfs/go-metrics-interface v0.0.1 // indirect + github.com/ipfs/go-unixfs v0.3.1 // indirect + github.com/ipfs/go-verifcid v0.0.1 // indirect github.com/ipld/go-codec-dagpb v1.3.2 // indirect - github.com/ipld/go-ipld-prime v0.16.0 // indirect + github.com/ipld/go-ipld-prime v0.18.0 // indirect + github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect github.com/jbenet/goprocess v0.1.4 // indirect - github.com/klauspost/cpuid/v2 v2.0.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/klauspost/cpuid/v2 v2.1.1 // indirect + github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/libp2p/go-libp2p v0.23.2 // indirect + github.com/libp2p/go-libp2p-core v0.16.1 // indirect + github.com/libp2p/go-libp2p-testing v0.12.0 // indirect + github.com/libp2p/go-openssl v0.1.0 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect + github.com/mattn/go-pointer v0.0.1 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect - github.com/multiformats/go-base32 v0.0.4 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.1.0 // indirect - github.com/multiformats/go-multibase v0.0.3 // indirect + github.com/multiformats/go-multiaddr v0.7.0 // indirect + github.com/multiformats/go-multibase v0.1.1 // indirect github.com/multiformats/go-varint v0.0.6 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect - github.com/rogpeppe/go-internal v1.8.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/sirupsen/logrus v1.8.1 // indirect + github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/tklauser/go-sysconf v0.3.5 // indirect - github.com/tklauser/numcpus v0.2.2 // indirect + github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799 // indirect + github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - go.uber.org/atomic v1.9.0 // indirect + go.opencensus.io v0.23.0 // indirect + go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.21.0 // indirect - golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect + go.uber.org/zap v1.23.0 // indirect + golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect + golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect - google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect - google.golang.org/grpc v1.45.0 // indirect - google.golang.org/protobuf v1.28.0 // indirect - gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect + google.golang.org/grpc v1.47.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools/v3 v3.4.0 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) diff --git a/extern/boostd-data/go.sum b/extern/boostd-data/go.sum index 5976194cc..cf22ef94e 100644 --- a/extern/boostd-data/go.sum +++ b/extern/boostd-data/go.sum @@ -1,8 +1,42 @@ +bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= +bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +contrib.go.opencensus.io/exporter/prometheus v0.4.0/go.mod h1:o7cosnyfuPVK0tB8q0QmaQNhGnptITnPQB+z1+qeFB0= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= @@ -10,28 +44,58 @@ dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ClickHouse/clickhouse-go v1.5.4/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= +github.com/GeertJohan/go.rice v1.0.2/go.mod h1:af5vUNlDNkCjOZeSGFgIJxDje9qdjsO6hshx0gTmZt4= +github.com/Gurpartap/async v0.0.0-20180927173644-4f7f499dd9ee/go.mod h1:W0GbEAA4uFNYOGG2cJpmFJ04E6SD1NLELPYZB57/7AY= +github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= -github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= +github.com/Kubuxu/imtui v0.0.0-20210401140320-41663d68d0fa/go.mod h1:WUmMvh9wMtqj1Xhf1hf3kp9RvL+y6odtdYxpyZjb90U= +github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo= +github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= +github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= +github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -39,28 +103,56 @@ github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.32.11/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= +github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= +github.com/benbjohnson/clock v1.0.2/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.2.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bep/debounce v1.2.0/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/briandowns/spinner v1.11.1/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= +github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= +github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= +github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= +github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= +github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= +github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= @@ -68,34 +160,67 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/buger/goterm v1.0.3/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/ceramicnetwork/go-dag-jose v0.1.0/go.mod h1:qYA1nYt0X8u4XoMAVoOV3upUVKtrxy/I670Dg5F0wjI= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/containerd v1.6.6 h1:xJNPhbrmz8xAMDNoVjHy9YHtWwEQNS+CDkcIRh7t8Y0= -github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/containerd v1.6.10 h1:8aiav7I2ZyQLbTlNMcBXyAU1FtFvp6VuyuW13qSd6Hk= +github.com/containerd/containerd v1.6.10/go.mod h1:CVqfxdJ95PDgORwA219AwwLrREZgrTFybXu2HfMKRG0= +github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.2.1-0.20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= github.com/couchbase/gocb/v2 v2.5.0 h1:BcNfLLeSs2pCppCK4wOW9aqDamQTSUMiwUZQCZvLnQo= github.com/couchbase/gocb/v2 v2.5.0/go.mod h1:xIF7+2G9uANWSJVv151NwM8lr+5cluxaOx20Fvlt8nw= github.com/couchbase/gocbcore/v10 v10.1.2 h1:u4OmKmub4soyf/cXMXaQohTXruJAxBL9wN6ke3DLe78= @@ -104,116 +229,261 @@ github.com/couchbaselabs/gocaves/client v0.0.0-20220223122017-22859b310bd2 h1:Ul github.com/couchbaselabs/gocaves/client v0.0.0-20220223122017-22859b310bd2/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= -github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= -github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= +github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= +github.com/dgraph-io/badger/v2 v2.2007.3/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/drand/bls12-381 v0.3.2/go.mod h1:dtcLgPtYT38L3NO6mPDYH0nbpc5tjPassDqiniuAt4Y= +github.com/drand/drand v1.3.0/go.mod h1:D6kAVlxufq1gi71YCGfzN455JrXF4Q272ZJEG975fzo= +github.com/drand/kyber v1.0.1-0.20200110225416-8de27ed8c0e2/go.mod h1:UpXoA0Upd1N9l4TvRPHr1qAUBBERj6JQ/mnKI3BPEmw= +github.com/drand/kyber v1.0.2/go.mod h1:x6KOpK7avKj0GJ4emhXFP5n7M7W7ChAPmnQh/OL6vRw= +github.com/drand/kyber v1.1.4/go.mod h1:9+IgTq7kadePhZg7eRwSD7+bA+bmvqRK+8DtmoV5a3U= +github.com/drand/kyber v1.1.7/go.mod h1:UkHLsI4W6+jT5PvNxmc0cvQAgppjTUpX+XCsN9TXmRo= +github.com/drand/kyber-bls12381 v0.2.0/go.mod h1:zQip/bHdeEB6HFZSU3v+d3cQE0GaBVQw9aR2E7AdoeI= +github.com/drand/kyber-bls12381 v0.2.1/go.mod h1:JwWn4nHO9Mp4F5qCie5sVIPQZ0X6cw8XAeMRvc/GXBE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/go-sysinfo v1.7.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0= +github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= +github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302/go.mod h1:qBlWZqWeVx9BjvqBsnC/8RUlAYpIFmPvgROcw0n1scE= +github.com/ema/qdisc v0.0.0-20190904071900-b82c76788043/go.mod h1:ix4kG2zvdUd8kEKSW0ZTr1XLks0epFpI4j745DXxlNE= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.10.19 h1:EOR5JbL4MD5yeOqv8W2iC1s4NximrTjqFccUz8lyBRA= -github.com/ethereum/go-ethereum v1.10.19/go.mod h1:IJBNMtzKcNHPtllYihy6BL2IgK1u+32JriaTbdt4v+w= +github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= +github.com/etclabscore/go-openrpc-reflect v0.0.36/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= +github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/filecoin-project/boost v1.4.0 h1:DFopqcEAjno+qNXyga5Ia05RFmWnaG0ishiWxdZzf70= +github.com/filecoin-project/boost v1.4.0/go.mod h1:MYMDsixQBYVhgx3gdBB7IWQ/AzfLVlN5oEixQlnhvo8= +github.com/filecoin-project/dagstore v0.5.2/go.mod h1:mdqKzYrRBHf1pRMthYfMv3n37oOw0Tkx7+TxPt240M0= +github.com/filecoin-project/dagstore v0.5.3/go.mod h1:mdqKzYrRBHf1pRMthYfMv3n37oOw0Tkx7+TxPt240M0= github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f/go.mod h1:+If3s2VxyjZn+KGGZIoRXBDSFQ9xL404JBJGf4WhEj0= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= -github.com/filecoin-project/go-address v1.1.0 h1:ofdtUtEsNxkIxkDw67ecSmvtzaVSdcea4boAmLbnHfE= -github.com/filecoin-project/go-address v1.1.0/go.mod h1:5t3z6qPmIADZBtuE9EIzi0EwzcRy2nVhpo0I/c1r0OA= +github.com/filecoin-project/go-address v0.0.6/go.mod h1:7B0/5DA13n6nHkB8bbGx1gWzG/dbTsZ0fgOJVGsM3TE= +github.com/filecoin-project/go-address v1.0.0 h1:IrexI0kpADLaPP+CdmU3CVAUqnW/FQC0KTmz4lVKiFU= +github.com/filecoin-project/go-address v1.0.0/go.mod h1:5t3z6qPmIADZBtuE9EIzi0EwzcRy2nVhpo0I/c1r0OA= github.com/filecoin-project/go-amt-ipld/v2 v2.1.0/go.mod h1:nfFPoGyX0CU9SkXX8EoCcSuHN1XcbN0c6KBh7yvP5fs= +github.com/filecoin-project/go-amt-ipld/v3 v3.0.0/go.mod h1:Qa95YNAbtoVCTSVtX38aAC1ptBnJfPma1R/zZsKmx4o= +github.com/filecoin-project/go-amt-ipld/v3 v3.1.0/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo= github.com/filecoin-project/go-amt-ipld/v4 v4.0.0/go.mod h1:gF053YQ4BIpzTNDoEwHZas7U3oAwncDVGvOHyY8oDpE= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= +github.com/filecoin-project/go-bitfield v0.2.3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.4/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= +github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= +github.com/filecoin-project/go-cbor-util v0.0.1/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-commp-utils v0.1.3/go.mod h1:3ENlD1pZySaUout0p9ANQrY3fDFoXdqyX04J+dWpK30= -github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082a837/go.mod h1:e2YBjSblNVoBckkbv3PPqsq71q98oFkFqL7s1etViGo= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= github.com/filecoin-project/go-crypto v0.0.1 h1:AcvpSGGCgjaY8y1az6AMfKQWreF/pWO2JJGLl6gCq6o= github.com/filecoin-project/go-crypto v0.0.1/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-dagaggregator-unixfs v0.2.0/go.mod h1:WTuJWgBQY0omnQqa8kRPT9O0Uj5wQOgslVMUuTeHdJ8= +github.com/filecoin-project/go-data-transfer v1.15.1/go.mod h1:dXsUoDjR9tKN7aV6R0BBDNNBPzbNvrrNuWt9MUn3yYc= +github.com/filecoin-project/go-data-transfer v1.15.2/go.mod h1:qXOJ3IF5dEJQHykXXTwcaRxu17bXAxr+LglXzkL6bZQ= +github.com/filecoin-project/go-ds-versioning v0.0.0-20211206185234-508abd7c2aff/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= +github.com/filecoin-project/go-ds-versioning v0.1.1/go.mod h1:C9/l9PnB1+mwPa26BBVpCjG/XQCB0yj/q5CK2J8X1I4= github.com/filecoin-project/go-fil-commcid v0.0.0-20200716160307-8f644712406f/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-commcid v0.0.0-20201016201715-d41df56b4f6a/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+h7aSPHc0cVqpSxA+TxIEpZQ= +github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= +github.com/filecoin-project/go-fil-markets v1.23.1/go.mod h1:V+1vFO34RZmpdECdikKGiyWhSNJK81Q89Kn0egA9iAk= +github.com/filecoin-project/go-fil-markets v1.23.2/go.mod h1:V+1vFO34RZmpdECdikKGiyWhSNJK81Q89Kn0egA9iAk= +github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= +github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0/go.mod h1:7aWZdaQ1b16BVoQUYR+eEvrDCGJoPLxFpDynFjYfBjI= +github.com/filecoin-project/go-hamt-ipld/v3 v3.0.1/go.mod h1:gXpNmr3oQx8l3o7qkGyDjJjYSRX7hp/FGOStdqrWyDI= github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0/go.mod h1:bxmzgT8tmeVQA1/gvBwFmYdT8SOFUwB3ovSUfG1Ux0g= +github.com/filecoin-project/go-indexer-core v0.2.16/go.mod h1:5kCKyhtT9k1vephr9l9SFGX8B/HowXIvOhGCkmbxwbY= +github.com/filecoin-project/go-jsonrpc v0.1.5/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= +github.com/filecoin-project/go-jsonrpc v0.1.8 h1:uXX/ikAk3Q4f/k8DRd9Zw+fWnfiYb5I+UI1tzlQgHog= +github.com/filecoin-project/go-jsonrpc v0.1.8/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= +github.com/filecoin-project/go-legs v0.4.4/go.mod h1:JQ3hA6xpJdbR8euZ2rO0jkxaMxeidXf0LDnVuqPAe9s= +github.com/filecoin-project/go-legs v0.4.9/go.mod h1:JQ3hA6xpJdbR8euZ2rO0jkxaMxeidXf0LDnVuqPAe9s= github.com/filecoin-project/go-padreader v0.0.0-20200903213702-ed5fae088b20/go.mod h1:mPn+LRRd5gEKNAtc+r3ScpW2JRU/pj4NBKdADYWHiak= +github.com/filecoin-project/go-padreader v0.0.1/go.mod h1:VYVPJqwpsfmtoHnAmPx6MUwmrK6HIcDqZJiuZhtmfLQ= +github.com/filecoin-project/go-paramfetch v0.0.4/go.mod h1:1FH85P8U+DUEmWk1Jkw3Bw7FrwTVUNHk/95PSPG+dts= github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I= +github.com/filecoin-project/go-state-types v0.0.0-20200928172055-2df22083d8ab/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.1.1-0.20210810190654-139e0e79e69e/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.1.3/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g= +github.com/filecoin-project/go-state-types v0.1.4/go.mod h1:xCA/WfKlC2zcn3fUmDv4IrzznwS98X5XW/irUP3Lhxg= +github.com/filecoin-project/go-state-types v0.1.5/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= +github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= +github.com/filecoin-project/go-state-types v0.1.10 h1:YrrJWWh2fU4VPhwHyPlDK5I4mB7bqgnRd3HCm9IOwIU= github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q= -github.com/filecoin-project/go-state-types v0.10.0-rc3 h1:qExCc2swTe5ndsiu9dEoMqIwppjuTNRbsAFgpzHnHbc= -github.com/filecoin-project/go-state-types v0.10.0-rc3/go.mod h1:aLIas+W8BWAfpLWEPUOGMPBdhcVwoCG4pIQSQk26024= +github.com/filecoin-project/go-statemachine v0.0.0-20200925024713-05bd7c71fbfe/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= +github.com/filecoin-project/go-statemachine v1.0.2-0.20220322104818-27f8fbb86dfd/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= +github.com/filecoin-project/go-statemachine v1.0.2/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54= +github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= +github.com/filecoin-project/go-statestore v0.2.0/go.mod h1:8sjBYbS35HwPzct7iT4lIXjLlYyPor80aU7t7a/Kspo= +github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8= +github.com/filecoin-project/index-provider v0.8.1/go.mod h1:c/Ym5HtWPp9NQgNc9dgSBMpSNsZ/DE9FEi9qVubl5RM= +github.com/filecoin-project/lotus v1.17.0/go.mod h1:hZ5L7E4uKWwp8E/8Bw3pBaai4wV7Utq/ZSbl9hIoFnU= +github.com/filecoin-project/pubsub v1.0.0/go.mod h1:GkpB33CcUtUNrLPhJgfdy4FDx4OMNR9k+46DHx/Lqrg= github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4= +github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= +github.com/filecoin-project/specs-actors v0.9.15-0.20220514164640-94e0d5e123bd/go.mod h1:pjGEe3QlWtK20ju/aFRsiArbMX6Cn8rqEhhsiCM9xYE= +github.com/filecoin-project/specs-actors v0.9.15/go.mod h1:pjGEe3QlWtK20ju/aFRsiArbMX6Cn8rqEhhsiCM9xYE= +github.com/filecoin-project/specs-actors/v2 v2.3.5-0.20210114162132-5b58b773f4fb/go.mod h1:LljnY2Mn2homxZsmokJZCpRuhOPxfXhvcek5gWkmqAc= +github.com/filecoin-project/specs-actors/v2 v2.3.6/go.mod h1:DJMpxVRXvev9t8P0XWA26RmTzN+MHiL9IlItVLT0zUc= +github.com/filecoin-project/specs-actors/v3 v3.1.0/go.mod h1:mpynccOLlIRy0QnR008BwYBwT9fen+sPR13MA1VmMww= +github.com/filecoin-project/specs-actors/v3 v3.1.2/go.mod h1:uOJn+m6W8OW/1mdWMEvxeM1cjQPxmps7s1Z4bJ9V4kY= +github.com/filecoin-project/specs-actors/v4 v4.0.0/go.mod h1:TkHXf/l7Wyw4ZejyXIPS2rK8bBO0rdwhTZyQQgaglng= +github.com/filecoin-project/specs-actors/v4 v4.0.2/go.mod h1:zT0GVFxwFS93prGK0b/rMd1sePjRQKfAuodQ9DFAd6Y= +github.com/filecoin-project/specs-actors/v5 v5.0.4/go.mod h1:5BAKRAMsOOlD8+qCw4UvT/lTLInCJ3JwOWZbX8Ipwq4= +github.com/filecoin-project/specs-actors/v5 v5.0.5/go.mod h1:dnfda2U+0ZidVnZJ44fnLMa3Mbyzwx51iW/brSsS+nc= +github.com/filecoin-project/specs-actors/v5 v5.0.6/go.mod h1:myb/UGwESp0V1f1BACXSUrFgTWLvGUoG0ZZH7eqriFM= +github.com/filecoin-project/specs-actors/v6 v6.0.0/go.mod h1:V1AYfi5GkHXipx1mnVivoICZh3wtwPxDVuds+fbfQtk= +github.com/filecoin-project/specs-actors/v6 v6.0.1/go.mod h1:V1AYfi5GkHXipx1mnVivoICZh3wtwPxDVuds+fbfQtk= +github.com/filecoin-project/specs-actors/v6 v6.0.2/go.mod h1:wnfVvPnYmzPZilNvSqCSSA/ZQX3rdV/U/Vf9EIoQhrI= +github.com/filecoin-project/specs-actors/v7 v7.0.0/go.mod h1:TA5FwCna+Yi36POaT7SLKXsgEDvJwc0V/L6ZsO19B9M= +github.com/filecoin-project/specs-actors/v7 v7.0.1/go.mod h1:tPLEYXoXhcpyLh69Ccq91SOuLXsPWjHiY27CzawjUEk= +github.com/filecoin-project/specs-actors/v8 v8.0.1/go.mod h1:UYIPg65iPWoFw5NEftREdJwv9b/5yaLKdCgTvNI/2FA= +github.com/filecoin-project/specs-storage v0.4.1/go.mod h1:Z2eK6uMwAOSLjek6+sy0jNV2DSsMEENziMUz0GHRFBw= +github.com/filecoin-project/storetheindex v0.4.17/go.mod h1:y2dL8C5D3PXi183hdxgGtM8vVYOZ1lg515tpl/D3tN8= +github.com/filecoin-project/test-vectors/schema v0.0.5/go.mod h1:iQ9QXLpYWL3m7warwvK1JC/pTri8mnfEmKygNDqqY6E= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= +github.com/gammazero/keymutex v0.0.2/go.mod h1:qtzWCCLMisQUmVa4dvqHVgwfh4BP2YB7JxNDGXnsKrs= +github.com/gammazero/radixtree v0.2.5/go.mod h1:VPqqCDZ3YZZxAzUUsIF/ytFBigVWV7JIV1Stld8hri0= +github.com/gbrlsnchs/jwt/v3 v3.0.1/go.mod h1:AncDcjXz18xetI3A6STfXq2w+LuTx8pQ8bGEwRN8zVM= +github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= +github.com/gdamore/tcell/v2 v2.2.0/go.mod h1:cTTuF84Dlj/RqmaCIV5p4w8uG1zWdk0SF6oBpwHp4fU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.4/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/spec v0.19.7/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.11/go.mod h1:vqK/dIdLGCosfvYsQV3WfC7N3TiZSnGY2RZKoFK7X28= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.8/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.11/go.mod h1:Uc0gKkdR+ojzsEpjh39QChyu92vPgIr72POcgHMAgSY= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= +github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -221,21 +491,38 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.0.3/go.mod h1:SavQ51ycCLnc7dGyJxp8YAmudx8xqiVrRf+6IXRsugc= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188/go.mod h1:vXjM/+wXQnTPR4KqTKDgJukSZ6amVRtWMPEjE6sQoK8= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -248,6 +535,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -256,21 +545,34 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -279,35 +581,66 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk= github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/graph-gophers/graphql-go v1.2.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-transport-ws v0.0.2/go.mod h1:5BVKvFzOd2BalVIBFfnfmHjpJi/MZ5rOj8G55mXvZ8g= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= +github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3vH7VqgtMxQ= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= +github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= +github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I8h3MITA53gN9OnWGCgaMa0JWVRdXthWw4M3CPM54OY= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -316,83 +649,179 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= +github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hodgesds/perf-utils v0.0.8/go.mod h1:F6TfvsbtrF88i++hou29dTXlI2sfsJv+gRZDtmTJkAs= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/hudl/fargo v1.4.0/go.mod h1:9Ai6uvFy5fQNq6VPKtg+Ceq1+eTY4nKUlR2JElEOcDo= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= +github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/iancoleman/orderedmap v0.1.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428/go.mod h1:uhpZMVGznybq1itEKXj6RYw9I71qK4kH+OGMjRC4KEo= +github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94/go.mod h1:GYeBD1CF7AqnKZK+UCytLcY3G+UKo0ByXX/3xfdNyqQ= +github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ= github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= +github.com/ipfs/go-bitswap v0.0.9/go.mod h1:kAPf5qgn2W2DrgAcscZ3HrM9qh4pH+X8Fkk3UPrwvis= +github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= +github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= +github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= +github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUiJ2teMvI= github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= +github.com/ipfs/go-bitswap v0.6.0 h1:f2rc6GZtoSFhEIzQmddgGiel9xntj02Dg0ZNf2hSC+w= +github.com/ipfs/go-bitswap v0.6.0/go.mod h1:Hj3ZXdOC5wBJvENtdqsixmzzRukqd8EHLxZLZc3mzRA= +github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= -github.com/ipfs/go-blockservice v0.2.1 h1:NJ4j/cwEfIg60rzAWcCIxRtOwbf6ZPK49MewNxObCPQ= +github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbRhbvNSdgc/7So= +github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= +github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= +github.com/ipfs/go-blockservice v0.1.7/go.mod h1:GmS+BAt4hrwBKkzE11AFDQUrnvqjwFatGS2MY7wOjEM= github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= +github.com/ipfs/go-blockservice v0.3.0 h1:cDgcZ+0P0Ih3sl8+qjFr2sVaMdysg/YZpLj5WJ8kiiw= +github.com/ipfs/go-blockservice v0.3.0/go.mod h1:P5ppi8IHDC7O+pA0AlGTF09jruB2h+oP3wVVaZl8sfk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= -github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0= github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= +github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s= +github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= +github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= +github.com/ipfs/go-datastore v0.4.2/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= -github.com/ipfs/go-datastore v0.5.1 h1:WkRhLuISI+XPD0uk3OskB0fYFSyqK8Ob5ZYew9Qa1nQ= github.com/ipfs/go-datastore v0.5.1/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= +github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= +github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= +github.com/ipfs/go-delegated-routing v0.2.2/go.mod h1:T8wrRhlXBHLPUR3bZQgArHPfdi7nBfOsZ1m5fr9tAp4= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= +github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= +github.com/ipfs/go-ds-badger v0.2.7/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= +github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= +github.com/ipfs/go-ds-badger2 v0.1.2/go.mod h1:3FtQmDv6fMubygEfU43bsFelYpIiXX/XEYA54l9eCwg= +github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= +github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= +github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE= +github.com/ipfs/go-fetcher v1.5.0/go.mod h1:5pDZ0393oRF/fHiLmtFZtpMNBQfHOYNPtryWedVuSWE= +github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo= +github.com/ipfs/go-filestore v1.1.0/go.mod h1:6e1/5Y6NvLuCRdmda/KA4GUhXJQ3Uat6vcWm2DJfxc8= +github.com/ipfs/go-filestore v1.2.0/go.mod h1:HLJrCxRXquTeEEpde4lTLMaE/MYJZD7WHLkp9z6+FF8= +github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= +github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc= +github.com/ipfs/go-graphsync v0.11.0/go.mod h1:wC+c8vGVjAHthsVIl8LKr37cUra2GOaMYcQNNmMxDqE= +github.com/ipfs/go-graphsync v0.13.1/go.mod h1:y8e8G6CmZeL9Srvx1l15CtGiRdf3h5JdQuqPz/iYL0A= github.com/ipfs/go-hamt-ipld v0.1.1/go.mod h1:1EZCr2v0jlCnhpa+aZ0JZYp8Tt2w16+JJOAVz17YcDk= +github.com/ipfs/go-ipfs v0.12.1/go.mod h1:Sbei4ScHevs2v47nUgONQMtHlUfaJjjTNDbhUU1OzOM= +github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= +github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= +github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= +github.com/ipfs/go-ipfs-blockstore v0.1.6/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= -github.com/ipfs/go-ipfs-blockstore v1.1.2 h1:WCXoZcMYnvOTmlpX+RSSnhVN0uCmbWTeepTGX5lgiXw= +github.com/ipfs/go-ipfs-blockstore v1.0.4-0.20210205083733-fb07d7bc5aec/go.mod h1:feuklK+m9POeWJzYQO7l05yNEgUiX5oELBNA8/Be33E= +github.com/ipfs/go-ipfs-blockstore v1.0.4/go.mod h1:uL7/gTJ8QIZ3MtA3dWf+s1a0U3fJy2fcEZAsovpRp+w= +github.com/ipfs/go-ipfs-blockstore v1.1.1/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY= github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY= +github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= +github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= +github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= +github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= +github.com/ipfs/go-ipfs-cmds v0.6.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk= +github.com/ipfs/go-ipfs-cmds v0.7.0/go.mod h1:y0bflH6m4g6ary4HniYt98UqbrVnRxmRarzeMdLIUn0= +github.com/ipfs/go-ipfs-config v0.5.3/go.mod h1:nSLCFtlaL+2rbl3F+9D4gQZQbT1LjRKx7TJg/IHz6oM= +github.com/ipfs/go-ipfs-config v0.18.0/go.mod h1:wz2lKzOjgJeYJa6zx8W9VT7mz+iSd0laBMqS/9wmX6A= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs= +github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoKnGyCcsoF6ueE= github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= +github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-interface v0.1.0 h1:TiMekCrOGQuWYtZO3mf4YJXDIdNgnKWZ9IE3fGlnWfo= github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI= -github.com/ipfs/go-ipfs-exchange-offline v0.1.1 h1:mEiXWdbMN6C7vtDG21Fphx8TGCbZPpQnz/496w/PL4g= +github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0= github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= +github.com/ipfs/go-ipfs-exchange-offline v0.2.0 h1:2PF4o4A7W656rC0RxuhUace997FTcDTcIQ6NoEtyjAI= +github.com/ipfs/go-ipfs-exchange-offline v0.2.0/go.mod h1:HjwBeW0dvZvfOMwDP0TSKXIHf2s+ksdP4E3MLDRtLKY= github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= +github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= +github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= +github.com/ipfs/go-ipfs-files v0.0.9/go.mod h1:aFv2uQ/qxWpL/6lidWvnSQmaVqCrf0TBGoUr+C1Fo84= +github.com/ipfs/go-ipfs-files v0.1.1 h1:/MbEowmpLo9PJTEQk16m9rKzUHjeP4KRU9nWJyJO324= +github.com/ipfs/go-ipfs-files v0.1.1/go.mod h1:8xkIrMWH+Y5P7HvJ4Yc5XWwIW2e52dyXUiC0tZyjDbM= +github.com/ipfs/go-ipfs-http-client v0.4.0/go.mod h1:NXzPUKt/QVCuR74a8angJCGOSLPImNi5LqaTxIep/70= +github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo= +github.com/ipfs/go-ipfs-pinner v0.2.1/go.mod h1:l1AtLL5bovb7opnG77sh4Y10waINz3Y1ni6CvTzx7oo= +github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= +github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= +github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= +github.com/ipfs/go-ipfs-provider v0.7.1/go.mod h1:QwdDYRYnC5sYGLlOwVDY/0ZB6T3zcMtu+5+GdGeUuw8= +github.com/ipfs/go-ipfs-routing v0.0.1/go.mod h1:k76lf20iKFxQTjcJokbPM9iBXVXVZhcOwc360N4nuKs= +github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= +github.com/ipfs/go-ipfs-routing v0.2.1 h1:E+whHWhJkdN9YeoHZNj5itzc+OR292AJ2uE9FFiW0BY= github.com/ipfs/go-ipfs-routing v0.2.1/go.mod h1:xiNNiwgjmLqPS1cimvAw6EyB9rkVDbiocA4yY+wRNLM= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= +github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= +github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.6-0.20211211231443-5d9b9e1f6fa8/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= @@ -400,52 +829,124 @@ github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eB github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= -github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= +github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= +github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ= +github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= +github.com/ipfs/go-ipld-git v0.1.1/go.mod h1:+VyMqF5lMcJh4rwEppV0e6g4nCCHXThLYYDpKUkJubI= github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc= github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= +github.com/ipfs/go-ipns v0.1.2/go.mod h1:ioQ0j02o6jdIVW+bmi18f4k2gRf0AV3kZ9KeHYHICnQ= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= +github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= github.com/ipfs/go-log v1.0.4/go.mod h1:oDCg2FkjogeFOhqqb+N39l2RpTNPL6F/StPkB3kPgcs= github.com/ipfs/go-log v1.0.5 h1:2dOuUCB1Z7uoczMWgAyDck5JLb72zHzrMnGnCNNbvY8= github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JPtIo= +github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= +github.com/ipfs/go-log/v2 v2.0.8/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.2-0.20200626104915-0016c0b4b3e4/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= +github.com/ipfs/go-log/v2 v2.1.2/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= +github.com/ipfs/go-log/v2 v2.4.0/go.mod h1:nPZnh7Cj7lwS3LpRU5Mwr2ol1c2gXIEXuF6aywqrtmo= +github.com/ipfs/go-log/v2 v2.5.0/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= -github.com/ipfs/go-merkledag v0.5.1 h1:tr17GPP5XtPhvPPiWtu20tSGZiZDuTaJRXBLcr79Umk= +github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKys/4GQQfto= +github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= +github.com/ipfs/go-merkledag v0.4.0/go.mod h1:XshXBkhyeS63YNGisLL1uDSfuTyrQIxVUOg3ojR5MOE= github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4= +github.com/ipfs/go-merkledag v0.6.0 h1:oV5WT2321tS4YQVOPgIrWHvJ0lJobRTerU+i9nmUCuA= +github.com/ipfs/go-merkledag v0.6.0/go.mod h1:9HSEwRd5sV+lbykiYP+2NC/3o6MZbKNaa4hfNcH5iH0= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnzdSH3u5UVlCdqSXnEks= +github.com/ipfs/go-mfs v0.2.1/go.mod h1:Woj80iuw4ajDnIP6+seRaoHpPsc9hmL0pk/nDNDWP88= +github.com/ipfs/go-namesys v0.4.0/go.mod h1:jpJwzodyP8DZdWN6DShRjVZw6gaqMr4nQLBSxU5cR6E= +github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno= +github.com/ipfs/go-path v0.0.9/go.mod h1:VpDkSBKQ9EFQOUgi54Tq/O/tGi8n1RfYNks13M3DEs8= +github.com/ipfs/go-path v0.1.1/go.mod h1:vC8q4AKOtrjJz2NnllIrmr2ZbGlF5fW2OKKyhV9ggb0= +github.com/ipfs/go-path v0.2.1/go.mod h1:NOScsVgxfC/eIw4nz6OiGwK42PjaSJ4Y/ZFPn1Xe07I= +github.com/ipfs/go-path v0.3.0/go.mod h1:NOScsVgxfC/eIw4nz6OiGwK42PjaSJ4Y/ZFPn1Xe07I= +github.com/ipfs/go-peertaskqueue v0.0.4/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ= +github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= +github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= +github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= +github.com/ipfs/go-peertaskqueue v0.7.1 h1:7PLjon3RZwRQMgOTvYccZ+mjzkmds/7YzSWKFlBAypE= +github.com/ipfs/go-peertaskqueue v0.7.1/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= +github.com/ipfs/go-pinning-service-http-client v0.1.0/go.mod h1:tcCKmlkWWH9JUUkKs8CrOZBanacNc1dmKLfjlyXAMu4= +github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= +github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= +github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= +github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0= +github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8= github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o= +github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s= +github.com/ipfs/go-unixfsnode v1.1.3/go.mod h1:ZZxUM5wXBC+G0Co9FjrYTOm+UlhZTjxLfRYdWY9veZ4= +github.com/ipfs/go-unixfsnode v1.4.0 h1:9BUxHBXrbNi8mWHc6j+5C580WJqtVw9uoeEKn4tMhwA= +github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= -github.com/ipld/go-car/v2 v2.1.2-0.20220124154420-9c7956a6eb9d h1:ZS+tQRulViCXtszhLwNZv44569uCjGiYlmR8Ks/tRBA= -github.com/ipld/go-car/v2 v2.1.2-0.20220124154420-9c7956a6eb9d/go.mod h1:5GmkU4R/nVUC/V/i2Ucl5iqa9373jZ3lqmvAKZnz4Vo= +github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o= +github.com/ipfs/interface-go-ipfs-core v0.5.2/go.mod h1:lNBJrdXHtWS46evMPBdWtDQMDsrKcGbxCOGoKLkztOE= +github.com/ipfs/interface-go-ipfs-core v0.7.0/go.mod h1:lF27E/nnSPbylPqKVXGZghal2hzifs3MmjyiEjnc9FY= +github.com/ipfs/iptb v1.4.0/go.mod h1:1rzHpCYtNp87/+hTxG5TfCVn/yMY3dKnLn8tBiMfdmg= +github.com/ipfs/iptb-plugins v0.3.0/go.mod h1:5QtOvckeIw4bY86gSH4fgh3p3gCSMn3FmIKr4gaBncA= +github.com/ipfs/tar-utils v0.0.2/go.mod h1:4qlnRWgTVljIMhSG2SqRYn66NT+3wrv/kZt9V+eqxDM= +github.com/ipld/edelweiss v0.1.2/go.mod h1:14NnBVHgrPO8cqDnKg7vc69LGI0aCAcax6mj21+99ec= +github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g= +github.com/ipld/go-car v0.3.2/go.mod h1:WEjynkVt04dr0GwJhry0KlaTeSDEiEYyMPOxDBQ17KE= +github.com/ipld/go-car v0.4.0/go.mod h1:Uslcn4O9cBKK9wqHm/cLTFacg6RAPv6LZx2mxd2Ypl4= +github.com/ipld/go-car v0.4.1-0.20220707083113-89de8134e58e/go.mod h1:Uslcn4O9cBKK9wqHm/cLTFacg6RAPv6LZx2mxd2Ypl4= +github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= +github.com/ipld/go-car/v2 v2.4.1/go.mod h1:zjpRf0Jew9gHqSvjsKVyoq9OY9SWoEKdYCQUKVaaPT0= +github.com/ipld/go-car/v2 v2.4.2-0.20220707083113-89de8134e58e h1:A4ttip3C2PLdE29/owgZAUgSX/qtIU6vphQU9CEPlN4= +github.com/ipld/go-car/v2 v2.4.2-0.20220707083113-89de8134e58e/go.mod h1:sDHqspWMwG6cC0lrid3Lq2xtIR4R6iy6ymCNT0drhaI= +github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= +github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY= github.com/ipld/go-codec-dagpb v1.3.2 h1:MZQUIjanHXXfDuYmtWYT8nFbqfFsZuyHClj6VDmSXr4= github.com/ipld/go-codec-dagpb v1.3.2/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= +github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= +github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.10.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.14.3-0.20211207234443-319145880958/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= -github.com/ipld/go-ipld-prime v0.16.0 h1:RS5hhjB/mcpeEPJvfyj0qbOj/QL+/j05heZ0qa97dVo= +github.com/ipld/go-ipld-prime v0.12.3/go.mod h1:PaeLYq8k6dJLmDUSLrzkEpoGV4PEfe/1OtFN/eALOc8= +github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= +github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.14.2/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.14.4-0.20211217152141-008fd70fc96f/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= +github.com/ipld/go-ipld-prime v0.17.0/go.mod h1:aYcKm5TIvGfY8P3QBKz/2gKcLxzJ1zDaD+o0bOowhgs= +github.com/ipld/go-ipld-prime v0.18.0 h1:xUk7NUBSWHEXdjiOu2sLXouFJOMs0yoYzeI5RAqhYQo= +github.com/ipld/go-ipld-prime v0.18.0/go.mod h1:735yXW548CKrLwVCYXzqx90p5deRJMVVxM9eJ4Qe+qE= +github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= +github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= +github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM= +github.com/ipld/go-storethehash v0.1.7/go.mod h1:O2CgbSwJfXCrYsjA1g1M7zJmVzzg71BM00ds6pyMLAQ= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= +github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c h1:uUx61FiAa1GI6ZmVd2wf2vULeQZIKG66eybjNXKYCz4= +github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c/go.mod h1:sdx1xVM9UuLw1tXnhJWN3piypTUO3vCIHYmG15KE/dU= github.com/jbenet/go-temp-err-catcher v0.0.0-20150120210811-aac704a3f4f2/go.mod h1:8GXXJV31xl8whumTzdZsTt3RnUIiPqzkyf7mxToRCMs= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8/go.mod h1:Ly/wlsjFq/qrU3Rar62tu1gASgGw6chQbSh/XgIIXCY= @@ -456,33 +957,64 @@ github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.1.1-0.20190114141812-62fb9bc030d1/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/jsimonetti/rtnetlink v0.0.0-20190830100107-3784a6c7c552/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3/go.mod h1:BYpt4ufZiIGv2nXn4gMxnfKV306n3mWXgNu/d2TqdTU= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kilic/bls12-381 v0.0.0-20200607163746-32e1441c8a9f/go.mod h1:XXfR6YFCRSrkEXbNlIyDsgXVNJWVUV30m/ebkVy9n6s= +github.com/kilic/bls12-381 v0.0.0-20200731194930-64c428e1bff5/go.mod h1:XXfR6YFCRSrkEXbNlIyDsgXVNJWVUV30m/ebkVy9n6s= +github.com/kilic/bls12-381 v0.0.0-20200820230200-6b2c19996391/go.mod h1:XXfR6YFCRSrkEXbNlIyDsgXVNJWVUV30m/ebkVy9n6s= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= +github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/koalacxr/quantile v0.0.1/go.mod h1:bGN/mCZLZ4lrSDHRQ6Lglj9chowGux8sGUIND+DQeD0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= +github.com/koron/go-ssdp v0.0.2/go.mod h1:XoLfkAiA2KeZsYh4DbHxD7h3nR2AZNqVQOa+LJuqPYs= +github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -494,90 +1026,248 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= +github.com/libp2p/go-addr-util v0.1.0/go.mod h1:6I3ZYuFr2O/9D+SoyM0zEw0EF3YkldtTX406BpdQMqw= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= +github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= +github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40JQWnayTvNMgD/vyk= +github.com/libp2p/go-conn-security-multistream v0.0.2/go.mod h1:nc9vud7inQ+d6SO0I/6dSWrdMnHnzZNHeyUQqrAJulE= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5Jb/P5ohUnFLNzEAR4DLSzpn2QLU= github.com/libp2p/go-conn-security-multistream v0.2.1/go.mod h1:cR1d8gA0Hr59Fj6NhaTpFhJZrjSYuNmhpT2r25zYR70= +github.com/libp2p/go-conn-security-multistream v0.3.0/go.mod h1:EEP47t4fw/bTelVmEzIDqSe69hO/ip52xBEhZMLWAHM= +github.com/libp2p/go-doh-resolver v0.3.1/go.mod h1:y5go1ZppAq9N2eppbX0xON01CyPBeUg2yS6BTssssog= +github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-eventbus v0.2.1/go.mod h1:jc2S4SoEVPP48H9Wpzm5aiGwUCBMfGhVhhBjyhhCJs8= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= +github.com/libp2p/go-flow-metrics v0.0.2/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= +github.com/libp2p/go-libp2p v0.0.30/go.mod h1:XWT8FGHlhptAv1+3V/+J5mEpzyui/5bvFsNuWYs611A= +github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= +github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= +github.com/libp2p/go-libp2p v0.3.1/go.mod h1:e6bwxbdYH1HqWTz8faTChKGR0BjPc8p+6SyP8GTTR7Y= +github.com/libp2p/go-libp2p v0.4.0/go.mod h1:9EsEIf9p2UDuwtPd0DwJsAl0qXVxgAnuDGRvHbfATfI= github.com/libp2p/go-libp2p v0.6.1/go.mod h1:CTFnWXogryAHjXAKEbOf1OWY+VeAP3lDMZkfEI5sT54= github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xSU1ivxn0k= github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw= github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= +github.com/libp2p/go-libp2p v0.12.0/go.mod h1:FpHZrfC1q7nA8jitvdjKBDF31hguaC676g/nT9PgQM0= +github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= +github.com/libp2p/go-libp2p v0.14.0/go.mod h1:dsQrWLAoIn+GkHPN/U+yypizkHiB9tnv79Os+kSgQ4Q= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= +github.com/libp2p/go-libp2p v0.14.4/go.mod h1:EIRU0Of4J5S8rkockZM7eJp2S0UrCyi55m2kJVru3rM= +github.com/libp2p/go-libp2p v0.16.0/go.mod h1:ump42BsirwAWxKzsCiFnTtN1Yc+DuPu76fyMX364/O4= +github.com/libp2p/go-libp2p v0.17.0/go.mod h1:Fkin50rsGdv5mm5BshBUtPRZknt9esfmYXBOYcwOTgw= +github.com/libp2p/go-libp2p v0.18.0/go.mod h1:+veaZ9z1SZQhmc5PW78jvnnxZ89Mgvmh4cggO11ETmw= +github.com/libp2p/go-libp2p v0.19.4/go.mod h1:MIt8y481VDhUe4ErWi1a4bvt/CjjFfOq6kZTothWIXY= +github.com/libp2p/go-libp2p v0.20.0/go.mod h1:g0C5Fu+aXXbCXkusCzLycuBowEih3ElmDqtbo61Em7k= +github.com/libp2p/go-libp2p v0.20.1/go.mod h1:XgJHsOhEBVBXp/2Sj9bm/yEyD94uunAaP6oaegdcKks= +github.com/libp2p/go-libp2p v0.20.3/go.mod h1:I+vndVanE/p/SjFbnA+BEmmfAUEpWxrdXZeyQ1Dus5c= +github.com/libp2p/go-libp2p v0.23.2 h1:yqyTeKQJyofWXxEv/eEVUvOrGdt/9x+0PIQ4N1kaxmE= +github.com/libp2p/go-libp2p v0.23.2/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= +github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= +github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= +github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= +github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= +github.com/libp2p/go-libp2p-autonat v0.0.6/go.mod h1:uZneLdOkZHro35xIhpbtTzLlgYturpu4J5+0cZK3MqE= +github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI= github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI= github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A= +github.com/libp2p/go-libp2p-autonat v0.4.0/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk= github.com/libp2p/go-libp2p-autonat v0.4.2/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk= +github.com/libp2p/go-libp2p-autonat v0.6.0/go.mod h1:bFC6kY8jwzNNWoqc8iGE57vsfwyJ/lP4O4DOV1e0B2o= +github.com/libp2p/go-libp2p-autonat v0.7.0/go.mod h1:uPvPn6J7cN+LCfFwW5tpOYvAz5NvPTc4iBamTV/WDMg= +github.com/libp2p/go-libp2p-autonat-svc v0.1.0/go.mod h1:fqi8Obl/z3R4PFVLm8xFtZ6PBL9MlV/xumymRFkKq5A= +github.com/libp2p/go-libp2p-blankhost v0.0.1/go.mod h1:Ibpbw/7cPPYwFb7PACIWdvxxv0t0XCCI10t7czjAjTc= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= +github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= github.com/libp2p/go-libp2p-blankhost v0.2.0/go.mod h1:eduNKXGTioTuQAUcZ5epXi9vMl+t4d8ugUBRQ4SqaNQ= +github.com/libp2p/go-libp2p-blankhost v0.3.0/go.mod h1:urPC+7U01nCGgJ3ZsV8jdwTp6Ji9ID0dMTvq+aJ+nZU= +github.com/libp2p/go-libp2p-circuit v0.0.9/go.mod h1:uU+IBvEQzCu953/ps7bYzC/D/R0Ho2A9LfKVVCatlqU= +github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= +github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= +github.com/libp2p/go-libp2p-circuit v0.1.3/go.mod h1:Xqh2TjSy8DD5iV2cCOMzdynd6h8OTBGoV1AWbWor3qM= github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo= github.com/libp2p/go-libp2p-circuit v0.4.0/go.mod h1:t/ktoFIUzM6uLQ+o1G6NuBl2ANhBKN9Bc8jRIk31MoA= +github.com/libp2p/go-libp2p-circuit v0.6.0/go.mod h1:kB8hY+zCpMeScyvFrKrGicRdid6vNXbunKE4rXATZ0M= +github.com/libp2p/go-libp2p-connmgr v0.1.1/go.mod h1:wZxh8veAmU5qdrfJ0ZBLcU8oJe9L82ciVP/fl1VHjXk= +github.com/libp2p/go-libp2p-connmgr v0.2.4/go.mod h1:YV0b/RIm8NGPnnNWM7hG9Q38OeQiQfKhHCCs1++ufn0= +github.com/libp2p/go-libp2p-connmgr v0.3.0/go.mod h1:RVoyPjJm0J9Vd1m6qUN2Tn7kJm4rL1Ml20pFsFgPGik= +github.com/libp2p/go-libp2p-connmgr v0.4.0/go.mod h1:exFQQm19PFAx+QuJmBPw4MM58QejzPJRFFFYnNmgi2w= github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco= +github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE= github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= +github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE= github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= +github.com/libp2p/go-libp2p-core v0.2.3/go.mod h1:GqhyQqyIAPsxFYXHMjfXgMv03lxsvM0mFzuYA9Ib42A= github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= +github.com/libp2p/go-libp2p-core v0.2.5/go.mod h1:6+5zJmKhsf7yHn1RbmYDu08qDUpIUxGdqHuEZckmZOA= github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= github.com/libp2p/go-libp2p-core v0.3.1/go.mod h1:thvWy0hvaSBhnVBaW37BvzgVV68OUhgJJLAa6almrII= github.com/libp2p/go-libp2p-core v0.4.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= github.com/libp2p/go-libp2p-core v0.5.0/go.mod h1:49XGI+kc38oGVwqSBhDEwytaAxgZasHhFfQKibzTls0= github.com/libp2p/go-libp2p-core v0.5.1/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= +github.com/libp2p/go-libp2p-core v0.5.2/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= +github.com/libp2p/go-libp2p-core v0.5.3/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= github.com/libp2p/go-libp2p-core v0.5.4/go.mod h1:uN7L2D4EvPCvzSH5SrhR72UWbnSGpt5/a35Sm4upn4Y= github.com/libp2p/go-libp2p-core v0.5.5/go.mod h1:vj3awlOr9+GMZJFH9s4mpt9RHHgGqeHCopzbYKZdRjM= github.com/libp2p/go-libp2p-core v0.5.6/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.5.7/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= github.com/libp2p/go-libp2p-core v0.6.0/go.mod h1:txwbVEhHEXikXn9gfC7/UDDw7rkxuX0bJvM49Ykaswo= +github.com/libp2p/go-libp2p-core v0.6.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.7.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.0/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.1/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.2/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= github.com/libp2p/go-libp2p-core v0.8.5/go.mod h1:FfewUH/YpvWbEB+ZY9AQRQ4TAD8sJBt/G1rVvhz5XT8= +github.com/libp2p/go-libp2p-core v0.8.6/go.mod h1:dgHr0l0hIKfWpGpqAMbpo19pen9wJfdCGv51mTmdpmM= +github.com/libp2p/go-libp2p-core v0.9.0/go.mod h1:ESsbz31oC3C1AvMJoGx26RTuCkNhmkSRCqZ0kQtJ2/8= +github.com/libp2p/go-libp2p-core v0.10.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= +github.com/libp2p/go-libp2p-core v0.11.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= +github.com/libp2p/go-libp2p-core v0.12.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= +github.com/libp2p/go-libp2p-core v0.13.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= +github.com/libp2p/go-libp2p-core v0.14.0/go.mod h1:tLasfcVdTXnixsLB0QYaT1syJOhsbrhG7q6pGrHtBg8= +github.com/libp2p/go-libp2p-core v0.15.1/go.mod h1:agSaboYM4hzB1cWekgVReqV5M4g5M+2eNNejV+1EEhs= +github.com/libp2p/go-libp2p-core v0.16.1 h1:bWoiEBqVkpJ13hbv/f69tHODp86t6mvc4fBN4DkK73M= +github.com/libp2p/go-libp2p-core v0.16.1/go.mod h1:O3i/7y+LqUb0N+qhzXjBjjpchgptWAVMG1Voegk7b4c= +github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= +github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= +github.com/libp2p/go-libp2p-daemon v0.2.2/go.mod h1:kyrpsLB2JeNYR2rvXSVWyY0iZuRIMhqzWR3im9BV6NQ= +github.com/libp2p/go-libp2p-discovery v0.0.5/go.mod h1:YtF20GUxjgoKZ4zmXj8j3Nb2TUSBHFlOCetzYdbZL5I= +github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= +github.com/libp2p/go-libp2p-discovery v0.6.0/go.mod h1:/u1voHt0tKIe5oIA1RHBKQLVCWPna2dXmPNHc2zR9S8= +github.com/libp2p/go-libp2p-discovery v0.7.0/go.mod h1:zPug0Rxib1aQG9iIdwOpRpBf18cAfZgzicO826UQP4I= +github.com/libp2p/go-libp2p-gostream v0.3.0/go.mod h1:pLBQu8db7vBMNINGsAwLL/ZCE8wng5V1FThoaE5rNjc= +github.com/libp2p/go-libp2p-gostream v0.3.1/go.mod h1:1V3b+u4Zhaq407UUY9JLCpboaeufAeVQbnvAt12LRsI= +github.com/libp2p/go-libp2p-gostream v0.4.0/go.mod h1:21DVGBcCQwRfEXZpCnZ2kG24QiEkBpEQvG53gYXE4u0= +github.com/libp2p/go-libp2p-gostream v0.4.1-0.20220720161416-e1952aede109/go.mod h1:21DVGBcCQwRfEXZpCnZ2kG24QiEkBpEQvG53gYXE4u0= +github.com/libp2p/go-libp2p-host v0.0.1/go.mod h1:qWd+H1yuU0m5CwzAkvbSjqKairayEHdR5MMl7Cwa7Go= +github.com/libp2p/go-libp2p-host v0.0.3/go.mod h1:Y/qPyA6C8j2coYyos1dfRm0I8+nvd4TGrDGt4tA7JR8= +github.com/libp2p/go-libp2p-http v0.2.1/go.mod h1:9KdioZ7XqNH0eZkZG9bulZLzHv11A7/12fT97agqWhg= +github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= +github.com/libp2p/go-libp2p-interface-connmgr v0.0.4/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= +github.com/libp2p/go-libp2p-interface-connmgr v0.0.5/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= +github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k= +github.com/libp2p/go-libp2p-kad-dht v0.2.1/go.mod h1:k7ONOlup7HKzQ68dE6lSnp07cdxdkmnRa+6B4Fh9/w0= +github.com/libp2p/go-libp2p-kad-dht v0.15.0/go.mod h1:rZtPxYu1TnHHz6n1RggdGrxUX/tA1C2/Wiw3ZMUDrU0= +github.com/libp2p/go-libp2p-kbucket v0.2.1/go.mod h1:/Rtu8tqbJ4WQ2KTCOMJhggMukOLNLNPY1EtEWWLxUvc= +github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= +github.com/libp2p/go-libp2p-kbucket v0.4.7/go.mod h1:XyVo99AfQH0foSf176k4jY1xUJ2+jUJIZCSDm7r2YKk= +github.com/libp2p/go-libp2p-loggables v0.0.1/go.mod h1:lDipDlBNYbpyqyPX/KcoO+eq0sJYEVR2JgOexcivchg= +github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= +github.com/libp2p/go-libp2p-metrics v0.0.1/go.mod h1:jQJ95SXXA/K1VZi13h52WZMa9ja78zjyy5rspMsC/08= +github.com/libp2p/go-libp2p-mplex v0.1.1/go.mod h1:KUQWpGkCzfV7UIpi8SKsAVxyBgz1c9R5EvxgnwLsb/I= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= github.com/libp2p/go-libp2p-mplex v0.2.2/go.mod h1:74S9eum0tVQdAfFiKxAyKzNdSuLqw5oadDq7+L/FELo= github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxWDkm+dVvjfuG3ek= +github.com/libp2p/go-libp2p-mplex v0.3.0/go.mod h1:l9QWxRbbb5/hQMECEb908GbS9Sm2UAR2KFZKUJEynEs= github.com/libp2p/go-libp2p-mplex v0.4.0/go.mod h1:yCyWJE2sc6TBTnFpjvLuEJgTSw/u+MamvzILKdX7asw= github.com/libp2p/go-libp2p-mplex v0.4.1/go.mod h1:cmy+3GfqfM1PceHTLL7zQzAAYaryDu6iPSC+CIb094g= +github.com/libp2p/go-libp2p-mplex v0.5.0/go.mod h1:eLImPJLkj3iG5t5lq68w3Vm5NAQ5BcKwrrb2VmOYb3M= +github.com/libp2p/go-libp2p-mplex v0.6.0/go.mod h1:i3usuPrBbh9FD2fLZjGpotyNkwr42KStYZQY7BeTiu4= +github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE= github.com/libp2p/go-libp2p-nat v0.0.6/go.mod h1:iV59LVhB3IkFvS6S6sauVTSOrNEANnINbI/fkaLimiw= +github.com/libp2p/go-libp2p-nat v0.1.0/go.mod h1:DQzAG+QbDYjN1/C3B6vXucLtz3u9rEonLVPtZVzQqks= +github.com/libp2p/go-libp2p-net v0.0.1/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c= +github.com/libp2p/go-libp2p-net v0.0.2/go.mod h1:Yt3zgmlsHOgUWSXmt5V/Jpz9upuJBE8EgNU9DrCcR8c= +github.com/libp2p/go-libp2p-netutil v0.0.1/go.mod h1:GdusFvujWZI9Vt0X5BKqwWWmZFxecf9Gt03cKxm2f/Q= +github.com/libp2p/go-libp2p-netutil v0.1.0 h1:zscYDNVEcGxyUpMd0JReUZTrpMfia8PmLKcKF72EAMQ= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= +github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM= github.com/libp2p/go-libp2p-noise v0.2.0/go.mod h1:IEbYhBBzGyvdLBoxxULL/SGbJARhUeqlO8lVSREYu2Q= +github.com/libp2p/go-libp2p-noise v0.3.0/go.mod h1:JNjHbociDJKHD64KTkzGnzqJ0FEV5gHJa6AB00kbCNQ= +github.com/libp2p/go-libp2p-noise v0.4.0/go.mod h1:BzzY5pyzCYSyJbQy9oD8z5oP2idsafjt4/X42h9DjZU= +github.com/libp2p/go-libp2p-noise v0.5.0/go.mod h1:CYYpJ+T4pox1d1J/hhUVxuMf6b2s3c41hFUgS8/yFQw= +github.com/libp2p/go-libp2p-peer v0.0.1/go.mod h1:nXQvOBbwVqoP+T5Y5nCjeH4sP9IX/J0AMzcDUVruVoo= +github.com/libp2p/go-libp2p-peer v0.1.1/go.mod h1:jkF12jGB4Gk/IOo+yomm+7oLWxF278F7UnrYUQ1Q8es= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= +github.com/libp2p/go-libp2p-peerstore v0.0.1/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= +github.com/libp2p/go-libp2p-peerstore v0.0.6/go.mod h1:RabLyPVJLuNQ+GFyoEkfi8H4Ti6k/HtZJ7YKgtSq+20= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= +github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ= github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= github.com/libp2p/go-libp2p-peerstore v0.2.6/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuDHItOpf2W8RxAi50P2s= +github.com/libp2p/go-libp2p-peerstore v0.2.8/go.mod h1:gGiPlXdz7mIHd2vfAsHzBNAMqSDkt2UBFwgcITgw1lA= +github.com/libp2p/go-libp2p-peerstore v0.4.0/go.mod h1:rDJUFyzEWPpXpEwywkcTYYzDHlwza8riYMaUzaN6hX0= +github.com/libp2p/go-libp2p-peerstore v0.6.0/go.mod h1:DGEmKdXrcYpK9Jha3sS7MhqYdInxJy84bIPtSu65bKc= +github.com/libp2p/go-libp2p-peerstore v0.7.0/go.mod h1:cdUWTHro83vpg6unCpGUr8qJoX3e93Vy8o97u5ppIM0= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= +github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= +github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= +github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= +github.com/libp2p/go-libp2p-pubsub v0.6.0/go.mod h1:nJv87QM2cU0w45KPR1rZicq+FmFIOD16zmT+ep1nOmg= +github.com/libp2p/go-libp2p-pubsub v0.7.0/go.mod h1:EuyBJFtF8qF67IEA98biwK8Xnw5MNJpJ/Z+8iWCMFwc= +github.com/libp2p/go-libp2p-pubsub v0.7.1/go.mod h1:EuyBJFtF8qF67IEA98biwK8Xnw5MNJpJ/Z+8iWCMFwc= +github.com/libp2p/go-libp2p-pubsub-router v0.5.0/go.mod h1:TRJKskSem3C0aSb3CmRgPwq6IleVFzds6hS09fmZbGM= +github.com/libp2p/go-libp2p-quic-transport v0.1.1/go.mod h1:wqG/jzhF3Pu2NrhJEvE+IE0NTHNXslOPn9JQzyCAxzU= github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= +github.com/libp2p/go-libp2p-quic-transport v0.11.2/go.mod h1:wlanzKtIh6pHrq+0U3p3DY9PJfGqxMgPaGKaK5LifwQ= +github.com/libp2p/go-libp2p-quic-transport v0.13.0/go.mod h1:39/ZWJ1TW/jx1iFkKzzUg00W6tDJh73FC0xYudjr7Hc= +github.com/libp2p/go-libp2p-quic-transport v0.15.0/go.mod h1:wv4uGwjcqe8Mhjj7N/Ic0aKjA+/10UnMlSzLO0yRpYQ= +github.com/libp2p/go-libp2p-quic-transport v0.15.2/go.mod h1:wv4uGwjcqe8Mhjj7N/Ic0aKjA+/10UnMlSzLO0yRpYQ= +github.com/libp2p/go-libp2p-quic-transport v0.16.0/go.mod h1:1BXjVMzr+w7EkPfiHkKnwsWjPjtfaNT0q8RS3tGDvEQ= +github.com/libp2p/go-libp2p-quic-transport v0.16.1/go.mod h1:1BXjVMzr+w7EkPfiHkKnwsWjPjtfaNT0q8RS3tGDvEQ= +github.com/libp2p/go-libp2p-quic-transport v0.17.0/go.mod h1:x4pw61P3/GRCcSLypcQJE/Q2+E9f4X+5aRcZLXf20LM= +github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= +github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= +github.com/libp2p/go-libp2p-record v0.1.2/go.mod h1:pal0eNcT5nqZaTV7UGhqeGqxFgGdsU/9W//C8dqjQDk= +github.com/libp2p/go-libp2p-record v0.1.3 h1:R27hoScIhQf/A8XJZ8lYpnqh9LatJ5YbHs28kCIfql0= +github.com/libp2p/go-libp2p-record v0.1.3/go.mod h1:yNUff/adKIfPnYQXgp6FQmNu3gLJ6EMg7+/vv2+9pY4= +github.com/libp2p/go-libp2p-resource-manager v0.1.5/go.mod h1:wJPNjeE4XQlxeidwqVY5G6DLOKqFK33u2n8blpl0I6Y= +github.com/libp2p/go-libp2p-resource-manager v0.2.1/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= +github.com/libp2p/go-libp2p-resource-manager v0.3.0/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= +github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= +github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE= +github.com/libp2p/go-libp2p-routing-helpers v0.2.3/go.mod h1:795bh+9YeoFl99rMASoiVgHdi5bjack0N1+AFAdbvBw= +github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= github.com/libp2p/go-libp2p-secio v0.2.2/go.mod h1:wP3bS+m5AUnFA+OFO7Er03uO1mncHG0uVwGrwvjYlNY= +github.com/libp2p/go-libp2p-swarm v0.0.6/go.mod h1:s5GZvzg9xXe8sbeESuFpjt8CJPTCa8mhEusweJqyFy8= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= +github.com/libp2p/go-libp2p-swarm v0.2.1/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHvokHKRhfkEgFKNM= github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0mdNSwckW7OYziM= github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= +github.com/libp2p/go-libp2p-swarm v0.3.1/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= +github.com/libp2p/go-libp2p-swarm v0.4.0/go.mod h1:XVFcO52VoLoo0eitSxNQWYq4D6sydGOweTOAjJNraCw= github.com/libp2p/go-libp2p-swarm v0.5.0/go.mod h1:sU9i6BoHE0Ve5SKz3y9WfKrh8dUat6JknzUehFx8xW4= +github.com/libp2p/go-libp2p-swarm v0.5.3/go.mod h1:NBn7eNW2lu568L7Ns9wdFrOhgRlkRnIDg0FLKbuu3i8= +github.com/libp2p/go-libp2p-swarm v0.8.0/go.mod h1:sOMp6dPuqco0r0GHTzfVheVBh6UEL0L1lXUZ5ot2Fvc= +github.com/libp2p/go-libp2p-swarm v0.9.0/go.mod h1:2f8d8uxTJmpeqHF/1ujjdXZp+98nNIbujVOMEZxCbZ8= +github.com/libp2p/go-libp2p-swarm v0.10.0/go.mod h1:71ceMcV6Rg/0rIQ97rsZWMzto1l9LnNquef+efcRbmA= +github.com/libp2p/go-libp2p-swarm v0.10.2/go.mod h1:Pdkq0QU5a+qu+oyqIV3bknMsnzk9lnNyKvB9acJ5aZs= +github.com/libp2p/go-libp2p-swarm v0.11.0/go.mod h1:sumjVYrC84gPSZOFKL8hNcnN6HZvJSwJ8ymaXeko4Lk= +github.com/libp2p/go-libp2p-testing v0.0.1/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= @@ -586,101 +1276,241 @@ github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eq github.com/libp2p/go-libp2p-testing v0.1.2-0.20200422005655-8775583591d8/go.mod h1:Qy8sAncLKpwXtS2dSnDOP8ktexIAHKu+J+pnZOFZLTc= github.com/libp2p/go-libp2p-testing v0.3.0/go.mod h1:efZkql4UZ7OVsEfaxNHZPzIehtsBXMrXnCfJIgDti5g= github.com/libp2p/go-libp2p-testing v0.4.0/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= +github.com/libp2p/go-libp2p-testing v0.4.2/go.mod h1:Q+PFXYoiYFN5CAEG2w3gLPEzotlKsNSbKQ/lImlOWF0= +github.com/libp2p/go-libp2p-testing v0.5.0/go.mod h1:QBk8fqIL1XNcno/l3/hhaIEn4aLRijpYOR+zVjjlh+A= +github.com/libp2p/go-libp2p-testing v0.6.0/go.mod h1:QBk8fqIL1XNcno/l3/hhaIEn4aLRijpYOR+zVjjlh+A= +github.com/libp2p/go-libp2p-testing v0.7.0/go.mod h1:OLbdn9DbgdMwv00v+tlp1l3oe2Cl+FAjoWIA2pa0X6E= +github.com/libp2p/go-libp2p-testing v0.8.0/go.mod h1:gRdsNxQSxAZowTgcLY7CC33xPmleZzoBpqSYbWenqPc= +github.com/libp2p/go-libp2p-testing v0.9.0/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= +github.com/libp2p/go-libp2p-testing v0.9.2/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= +github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= +github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= +github.com/libp2p/go-libp2p-tls v0.3.0/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= +github.com/libp2p/go-libp2p-tls v0.3.1/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= +github.com/libp2p/go-libp2p-tls v0.4.1/go.mod h1:EKCixHEysLNDlLUoKxv+3f/Lp90O2EXNjTr0UQDnrIw= +github.com/libp2p/go-libp2p-tls v0.5.0/go.mod h1:1a4tq0xQSZ0kAkDkZVAppuP3SAIUHcnzi2djJ/2EN4I= +github.com/libp2p/go-libp2p-transport v0.0.1/go.mod h1:UzbUs9X+PHOSw7S3ZmeOxfnwaQY5vGDzZmKPod3N3tk= +github.com/libp2p/go-libp2p-transport v0.0.5/go.mod h1:StoY3sx6IqsP6XKoabsPnHCwqKXWUMWU7Rfcsubee/A= +github.com/libp2p/go-libp2p-transport-upgrader v0.0.4/go.mod h1:RGq+tupk+oj7PzL2kn/m1w6YXxcIAYJYeI90h6BGgUc= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= +github.com/libp2p/go-libp2p-transport-upgrader v0.4.0/go.mod h1:J4ko0ObtZSmgn5BX5AmegP+dK3CSnU2lMCKsSq/EY0s= github.com/libp2p/go-libp2p-transport-upgrader v0.4.2/go.mod h1:NR8ne1VwfreD5VIWIU62Agt/J18ekORFU/j1i2y8zvk= +github.com/libp2p/go-libp2p-transport-upgrader v0.4.3/go.mod h1:bpkldbOWXMrXhpZbSV1mQxTrefOg2Fi+k1ClDSA4ppw= +github.com/libp2p/go-libp2p-transport-upgrader v0.4.6/go.mod h1:JE0WQuQdy+uLZ5zOaI3Nw9dWGYJIA7mywEtP2lMvnyk= +github.com/libp2p/go-libp2p-transport-upgrader v0.5.0/go.mod h1:Rc+XODlB3yce7dvFV4q/RmyJGsFcCZRkeZMu/Zdg0mo= +github.com/libp2p/go-libp2p-transport-upgrader v0.6.0/go.mod h1:1e07y1ZSZdHo9HPbuU8IztM1Cj+DR5twgycb4pnRzRo= +github.com/libp2p/go-libp2p-transport-upgrader v0.7.0/go.mod h1:GIR2aTRp1J5yjVlkUoFqMkdobfob6RnAwYg/RZPhrzg= +github.com/libp2p/go-libp2p-transport-upgrader v0.7.1/go.mod h1:GIR2aTRp1J5yjVlkUoFqMkdobfob6RnAwYg/RZPhrzg= +github.com/libp2p/go-libp2p-xor v0.0.0-20210714161855-5c005aca55db/go.mod h1:LSTM5yRnjGZbWNTA/hRwq2gGFrvRIbQJscoIL/u6InY= +github.com/libp2p/go-libp2p-yamux v0.1.2/go.mod h1:xUoV/RmYkg6BW/qGxA9XJyg+HzXFYkeXbnhjmnYzKp8= +github.com/libp2p/go-libp2p-yamux v0.1.3/go.mod h1:VGSQVrqkh6y4nm0189qqxMtvyBft44MOYYPpYKXiVt4= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= +github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI= github.com/libp2p/go-libp2p-yamux v0.2.2/go.mod h1:lIohaR0pT6mOt0AZ0L2dFze9hds9Req3OfS+B+dv4qw= github.com/libp2p/go-libp2p-yamux v0.2.5/go.mod h1:Zpgj6arbyQrmZ3wxSZxfBmbdnWtbZ48OpsfmQVTErwA= github.com/libp2p/go-libp2p-yamux v0.2.7/go.mod h1:X28ENrBMU/nm4I3Nx4sZ4dgjZ6VhLEn0XhIoZ5viCwU= github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2EzI2h7HbFm9eAKI4= github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30= github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po= +github.com/libp2p/go-libp2p-yamux v0.5.1/go.mod h1:dowuvDu8CRWmr0iqySMiSxK+W0iL5cMVO9S94Y6gkv4= +github.com/libp2p/go-libp2p-yamux v0.5.3/go.mod h1:Vy3TMonBAfTMXHWopsMc8iX/XGRYrRlpUaMzaeuHV/s= github.com/libp2p/go-libp2p-yamux v0.5.4/go.mod h1:tfrXbyaTqqSU654GTvK3ocnSZL3BuHoeTSqhcel1wsE= +github.com/libp2p/go-libp2p-yamux v0.6.0/go.mod h1:MRhd6mAYnFRnSISp4M8i0ClV/j+mWHo2mYLifWGw33k= +github.com/libp2p/go-libp2p-yamux v0.7.0/go.mod h1:fMyA0CsPfHkIuBU0wjRGrCjTBFiXTXxG0k5M4ETv+08= +github.com/libp2p/go-libp2p-yamux v0.8.0/go.mod h1:yTkPgN2ib8FHyU1ZcVD7aelzyAqXXwEPbyx+aSKm9h8= +github.com/libp2p/go-libp2p-yamux v0.8.1/go.mod h1:rUozF8Jah2dL9LLGyBaBeTQeARdwhefMCTQVQt6QobE= +github.com/libp2p/go-libp2p-yamux v0.8.2/go.mod h1:rUozF8Jah2dL9LLGyBaBeTQeARdwhefMCTQVQt6QobE= +github.com/libp2p/go-libp2p-yamux v0.9.1/go.mod h1:wRc6wvyxQINFcKe7daL4BeQ02Iyp+wxyC8WCNfngBrA= +github.com/libp2p/go-maddr-filter v0.0.1/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.4/go.mod h1:6eT12kSQMA9x2pvFQa+xesMKUBlj9VImZbj3B9FBH/Q= github.com/libp2p/go-maddr-filter v0.0.5/go.mod h1:Jk+36PMfIqCJhAnaASRH83bdAvfDRp/w6ENFaC9bG+M= github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= github.com/libp2p/go-mplex v0.0.3/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= +github.com/libp2p/go-mplex v0.0.4/go.mod h1:pK5yMLmOoBR1pNCqDlA2GQrdAVTMkqFalaTWe7l4Yd0= github.com/libp2p/go-mplex v0.1.0/go.mod h1:SXgmdki2kwCUlCCbfGLEgHjC4pFqhTp0ZoV6aiKgxDU= github.com/libp2p/go-mplex v0.1.1/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= github.com/libp2p/go-mplex v0.1.2/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3SFXI1lk= github.com/libp2p/go-mplex v0.2.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= +github.com/libp2p/go-mplex v0.4.0/go.mod h1:y26Lx+wNVtMYMaPu300Cbot5LkEZ4tJaNYeHeT9dh6E= +github.com/libp2p/go-mplex v0.6.0/go.mod h1:y26Lx+wNVtMYMaPu300Cbot5LkEZ4tJaNYeHeT9dh6E= +github.com/libp2p/go-mplex v0.7.0/go.mod h1:rW8ThnRcYWft/Jb2jeORBmPd6xuG3dGxWN/W168L9EU= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= +github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= +github.com/libp2p/go-msgio v0.1.0/go.mod h1:eNlv2vy9V2X/kNldcZ+SShFE++o2Yjxwx6RAYsmgJnE= +github.com/libp2p/go-msgio v0.2.0 h1:W6shmB+FeynDrUVl2dgFQvzfBZcXiyqY4VmpQLu9FqU= +github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= +github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= +github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= +github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= github.com/libp2p/go-netroute v0.1.6/go.mod h1:AqhkMh0VuWmfgtxKPp3Oc1LdU5QSWS7wl0QLhSZqXxQ= +github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4nWRE= +github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= +github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= +github.com/libp2p/go-reuseport v0.1.0/go.mod h1:bQVn9hmfcTaoo0c9v5pBhOarsU1eNOBZdaAd2hzXRKU= +github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= +github.com/libp2p/go-reuseport-transport v0.0.5/go.mod h1:TC62hhPc8qs5c/RoXDZG6YmjK+/YWUPC0yYmeUecbjc= +github.com/libp2p/go-reuseport-transport v0.1.0/go.mod h1:vev0C0uMkzriDY59yFHD9v+ujJvYmDQVLowvAjEOmfw= github.com/libp2p/go-sockaddr v0.0.2/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-sockaddr v0.1.0/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/libp2p/go-sockaddr v0.1.1/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= +github.com/libp2p/go-socket-activation v0.1.0/go.mod h1:gzda2dNkMG5Ti2OfWNNwW0FDIbj0g/aJJU320FcLfhk= github.com/libp2p/go-stream-muxer v0.0.1/go.mod h1:bAo8x7YkSpadMTbtTaxGVHWUQsR/l5MEaHbKaliuT14= +github.com/libp2p/go-stream-muxer v0.1.0/go.mod h1:8JAVsjeRBCWwPoZeH0W1imLOcriqXJyFvB0mR4A04sQ= +github.com/libp2p/go-stream-muxer-multistream v0.1.1/go.mod h1:zmGdfkQ1AzOECIAcccoL8L//laqawOsO03zX8Sa+eGw= github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= github.com/libp2p/go-stream-muxer-multistream v0.3.0/go.mod h1:yDh8abSIzmZtqtOt64gFJUXEryejzNb0lisTt+fAMJA= +github.com/libp2p/go-stream-muxer-multistream v0.4.0/go.mod h1:nb+dGViZleRP4XcyHuZSVrJCBl55nRBOMmiSL/dyziw= +github.com/libp2p/go-tcp-transport v0.0.4/go.mod h1:+E8HvC8ezEVOxIo3V5vCK9l1y/19K427vCzQ+xHKH/o= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY= github.com/libp2p/go-tcp-transport v0.2.0/go.mod h1:vX2U0CnWimU4h0SGSEsg++AzvBcroCGYw28kh94oLe0= +github.com/libp2p/go-tcp-transport v0.2.1/go.mod h1:zskiJ70MEfWz2MKxvFB/Pv+tPIB1PpPUrHIWQ8aFw7M= github.com/libp2p/go-tcp-transport v0.2.3/go.mod h1:9dvr03yqrPyYGIEN6Dy5UvdJZjyPFvl1S/igQ5QD1SU= +github.com/libp2p/go-tcp-transport v0.2.4/go.mod h1:9dvr03yqrPyYGIEN6Dy5UvdJZjyPFvl1S/igQ5QD1SU= +github.com/libp2p/go-tcp-transport v0.2.7/go.mod h1:lue9p1b3VmZj1MhhEGB/etmvF/nBQ0X9CW2DutBT3MM= +github.com/libp2p/go-tcp-transport v0.4.0/go.mod h1:0y52Rwrn4076xdJYu/51/qJIdxz+EWDAOG2S45sV3VI= +github.com/libp2p/go-tcp-transport v0.5.0/go.mod h1:UPPL0DIjQqiWRwVAb+CEQlaAG0rp/mCqJfIhFcLHc4Y= +github.com/libp2p/go-tcp-transport v0.5.1/go.mod h1:UPPL0DIjQqiWRwVAb+CEQlaAG0rp/mCqJfIhFcLHc4Y= +github.com/libp2p/go-testutil v0.0.1/go.mod h1:iAcJc/DKJQanJ5ws2V+u5ywdL2n12X1WbbEG+Jjy69I= +github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= +github.com/libp2p/go-ws-transport v0.0.5/go.mod h1:Qbl4BxPfXXhhd/o0wcrgoaItHqA9tnZjoFZnxykuaXU= +github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= +github.com/libp2p/go-ws-transport v0.1.2/go.mod h1:dsh2Ld8F+XNmzpkaAijmg5Is+e9l6/1tK/6VFOdN69Y= github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM= github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= +github.com/libp2p/go-ws-transport v0.3.1/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA= +github.com/libp2p/go-ws-transport v0.5.0/go.mod h1:I2juo1dNTbl8BKSBYo98XY85kU2xds1iamArLvl8kNg= +github.com/libp2p/go-ws-transport v0.6.0/go.mod h1:dXqtI9e2JV9FtF1NOtWVZSKXh5zXvnuwPXfj8GPBbYU= +github.com/libp2p/go-yamux v1.2.1/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= +github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.5/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= +github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U= +github.com/libp2p/go-yamux/v2 v2.1.1/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ= github.com/libp2p/go-yamux/v2 v2.2.0/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ= +github.com/libp2p/go-yamux/v2 v2.3.0/go.mod h1:iTU+lOIn/2h0AgKcL49clNTwfEw+WSfDYrXe05EyKIs= +github.com/libp2p/go-yamux/v3 v3.0.1/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo= +github.com/libp2p/go-yamux/v3 v3.0.2/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo= +github.com/libp2p/go-yamux/v3 v3.1.1/go.mod h1:jeLEQgLXqE2YqX1ilAClIfCMDY+0uXQUKmmb/qp0gT4= +github.com/libp2p/go-yamux/v3 v3.1.2/go.mod h1:jeLEQgLXqE2YqX1ilAClIfCMDY+0uXQUKmmb/qp0gT4= +github.com/libp2p/zeroconf/v2 v2.1.1/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= +github.com/lucas-clemente/quic-go v0.21.2/go.mod h1:vF5M1XqhBAHgbjKcJOXY3JZz3GP0T3FQhz/uyOUS38Q= +github.com/lucas-clemente/quic-go v0.23.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= +github.com/lucas-clemente/quic-go v0.24.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= +github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg= +github.com/lucas-clemente/quic-go v0.27.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= +github.com/lucas-clemente/quic-go v0.27.1/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= +github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= +github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs= github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-15 v0.1.5/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I= +github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= +github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= +github.com/marten-seemann/qtls-go1-17 v0.1.0-rc.1/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8= +github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8= +github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= +github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI= +github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= +github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-xmlrpc v0.0.3/go.mod h1:mqc2dz7tP5x5BKlCahN/n+hs7OSZKJkS9JsHNBRlrxA= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= +github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= +github.com/mdlayher/netlink v0.0.0-20190828143259-340058475d09/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= +github.com/mdlayher/wifi v0.0.0-20190303161829-b1436901ddee/go.mod h1:Evt/EIne46u9PtQbeTx2NTcqURpr5K4SvKtGmBuDPN8= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= @@ -689,19 +1519,28 @@ github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= @@ -710,9 +1549,11 @@ github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjW github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= -github.com/multiformats/go-base32 v0.0.4 h1:+qMh4a2f37b4xTNs6mqitDinryCI+tfO2dRVMN9mjSE= github.com/multiformats/go-base32 v0.0.4/go.mod h1:jNLFzjPZtp3aIARHbJRZIaPuspdH0J6q39uUM5pnABM= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.1.0 h1:JR6TyF7JjGd3m6FbLU2cOxhC0Li8z8dLNGQ89tUg4F4= github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= @@ -726,11 +1567,21 @@ github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u github.com/multiformats/go-multiaddr v0.3.0/go.mod h1:dF9kph9wfJ+3VLAaeBqo9Of8x4fJxp6ggJGteB8HQTI= github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWzmBD8uLLL4bXW0XfGc= github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= +github.com/multiformats/go-multiaddr v0.4.0/go.mod h1:YcpyLH8ZPudLxQlemYBPhSm0/oCXAT8Z4mzFpyoPyRc= +github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= +github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar+bR+gh4zgbfr3SNug= +github.com/multiformats/go-multiaddr v0.7.0 h1:gskHcdaCyPtp9XskVwtvEeQOG465sCohbQIirSyqxrc= +github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.1.0/go.mod h1:01k2RAqtoXIuPa3DCavAE9/6jc6nM0H3EgZyfUhN2oY= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= +github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ= +github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= +github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= github.com/multiformats/go-multiaddr-net v0.1.0/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= @@ -741,25 +1592,41 @@ github.com/multiformats/go-multiaddr-net v0.1.4/go.mod h1:ilNnaM9HbmVFqsb/qcNysj github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= -github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= +github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= +github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.4.1 h1:BSJbf+zpghcZMZrwTYBGwy0CPcVZGWiC72Cp8bBd4R4= +github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= +github.com/multiformats/go-multicodec v0.6.0 h1:KhH2kSuCARyuJraYMFxrNO3DqIaYhOdS039kbhgVwpE= +github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= -github.com/multiformats/go-multihash v0.1.0 h1:CgAgwqk3//SVEw3T+6DqI4mWMyRuDwZtOWcJT0q9+EA= +github.com/multiformats/go-multihash v0.0.16/go.mod h1:zhfEIgVnB/rPMfxgFw15ZmGoNaKyNUIE4IWHG/kC+Ag= github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= +github.com/multiformats/go-multihash v0.2.0/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= +github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= +github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= +github.com/multiformats/go-multistream v0.2.0/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= github.com/multiformats/go-multistream v0.2.2/go.mod h1:UIcnm7Zuo8HKG+HkWgfQsGL+/MIEhyTqbODbIUwSXKs= +github.com/multiformats/go-multistream v0.3.0/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= +github.com/multiformats/go-multistream v0.3.1/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= +github.com/multiformats/go-multistream v0.3.3 h1:d5PZpjwRgVlbwfdTDjife7XszfZd8KYWfROYFlGcR8o= +github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -769,18 +1636,29 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/jwt v1.2.2/go.mod h1:/xX356yQA6LuXI9xWW7mZNpxgF2mBmGecH+Fj34sP5Q= +github.com/nats-io/jwt/v2 v2.0.3/go.mod h1:VRP+deawSXyhNjXmxPCHskrR6Mq50BqpEI5SEcNiGlY= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats-server/v2 v2.5.0/go.mod h1:Kj86UtrXAL6LwYRA6H4RqzkHhK0Vcv2ZnKD5WbQ1t3g= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nats.go v1.12.1/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.2.0/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= +github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c/go.mod h1:7qN3Y0BvzRUf4LofcoJplQL10lsFDb4PYlePTVwrP28= +github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -788,6 +1666,8 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -795,14 +1675,26 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= -github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= +github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= +github.com/opentracing-contrib/go-grpc v0.0.0-20191001143057-db30781987df/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= +github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -813,14 +1705,21 @@ github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTm github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE= +github.com/ory/dockertest/v3 v3.8.1/go.mod h1:wSRQ3wmkz+uSARYMk7kVJFDBGm8x5gSxIhI7NDc+BAQ= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM= +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -830,18 +1729,29 @@ github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6J github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/pressly/goose/v3 v3.5.3/go.mod h1:IL4NNMdXx9O6hHpGbNB5l1hkVe/Avoz4gBDE5g7rQNg= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -849,38 +1759,66 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289/go.mod h1:FGbBv5OPKjch+jNUJmEQpMZytIdyW0NdBtWFcfSKusc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/statsd_exporter v0.21.0/go.mod h1:rbT83sZq2V+p73lHhPZfMc3MLCHmSHelCh9hSGYNLTQ= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= +github.com/raulk/go-watchdog v1.2.0/go.mod h1:lzSbAl5sh4rtI8tYHU01BWIDzgzqaQLj6RcA1i4mlqI= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/sercand/kuberesolver v2.1.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= +github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= @@ -904,6 +1842,7 @@ github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go. github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= +github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745/go.mod h1:G81aIFAMS9ECrwBYR9YxhlPjWgrItd+Kje78O6+uqm8= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -911,18 +1850,22 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a/go.mod h1:LeFCbQYJ3KJlPs/FvPz2dy1tkpxyeNESVyCNNzRXFR0= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -931,117 +1874,219 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= +github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= -github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= -github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= +github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v1.5.1-0.20181102163054-1fc5c315e03c/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/urfave/cli/v2 v2.8.1/go.mod h1:Z41J9TPoffeoqP0Iza0YbAhGvymRdZAd2uPmZ5JxRdY= github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU= github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= +github.com/warpfork/go-testmark v0.9.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= +github.com/warpfork/go-testmark v0.10.0 h1:E86YlUMYfwIacEsQGlnTvjk1IgYkyTGjPhF0RnwTCmw= +github.com/warpfork/go-testmark v0.10.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/weaveworks/common v0.0.0-20200512154658-384f10054ec5/go.mod h1:c98fKi5B9u8OsKGiWHLRKus6ToQ1Tubeow44ECO1uxY= +github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= +github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= +github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba/go.mod h1:CHQnYnQUEPydYCwuy8lmTHfGmdw9TKrhWV0xLx8l0oM= +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200414195334-429a0b5e922e/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20200504204219-64967432584d/go.mod h1:W5MvapuoHRP8rz4vxjwCK1pDqF1aQcWsV5PZ+AHbqdg= +github.com/whyrusleeping/cbor-gen v0.0.0-20200710004633-5379fc63235d/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200715143311-227fab5a2377/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200723185710-6a3894a6352b/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200810223238-211df3b9e24c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200826160007-0b9f6c5fb163/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210118024343-169e9d70c0c2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20210303213153-67a261a1d291/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20220302191723-37c43cae8e14/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799 h1:DOOT2B85S0tHoLGTzV+FakaSSihgRCVwZkjqKQP5L/w= github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= +github.com/whyrusleeping/go-ctrlnet v0.0.0-20180313164037-f564fbbdaa95/go.mod h1:SJqKCCPXRfBFCwXjfNT/skfsceF7+MBFLI2OrvuRA7g= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= +github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= +github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1/go.mod h1:tKH72zYNt/exx6/5IQO6L9LoQ0rEjd5SbbWaDTs9Zso= +github.com/whyrusleeping/ledger-filecoin-go v0.9.1-0.20201010031517-c3dcc1bddce4/go.mod h1:K+EVq8d5QcQ2At5VECsA+SNZvWefyBXh8TnIsxo1OvQ= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= +github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/c-for-go v0.0.0-20200718154222-87b0065af829/go.mod h1:h/1PEBwj7Ym/8kOuMWvO2ujZ6Lt+TMbySEXNhjjR87I= +github.com/xlab/c-for-go v0.0.0-20201112171043-ea6dce5809cb/go.mod h1:pbNsDSxn1ICiNn9Ct4ZGNrwzfkkwYbx/lw8VuyutFIg= github.com/xlab/pkgconfig v0.0.0-20170226114623-cea12a0fd245/go.mod h1:C+diUUz7pxhNY6KAoLgrTYARGWnt82zWTylZlxT92vk= github.com/xorcare/golden v0.6.0/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= +github.com/xorcare/golden v0.6.1-0.20191112154924-b87f686d7542/go.mod h1:7T39/ZMvaSEZlBPoYfVFmsBLmUl3uz9IuzWj/U6FtvQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/ybbus/jsonrpc/v2 v2.1.6/go.mod h1:rIuG1+ORoiqocf9xs/v+ecaAVeo3zcZHQgInyKFMeg0= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= +github.com/zondax/ledger-go v0.12.1/go.mod h1:KatxXrVDzgWwbssUWsF5+cOJHXPvzQ09YSlzGNuhOEo= +go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= +go.dedis.ch/kyber/v3 v3.0.4/go.mod h1:OzvaEnPvKlyrWyp3kGXlFdp7ap1VC6RkZDTaPikqhsQ= +go.dedis.ch/kyber/v3 v3.0.9/go.mod h1:rhNjUUg6ahf8HEg5HUvVBYoWY4boAafX8tYxX+PS+qg= +go.dedis.ch/protobuf v1.0.5/go.mod h1:eIV4wicvi6JK0q/QnfIEGeSFNG0ZeB24kzut5+HaRLo= +go.dedis.ch/protobuf v1.0.7/go.mod h1:pv5ysfkDX/EawiPqcW3ikOxsL5t+BqnV6xHSmE79KI4= +go.dedis.ch/protobuf v1.0.11/go.mod h1:97QR256dnkimeNdfmURz0wAMNVbd1VmLXhG1CrTYrJ4= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.6-0.20201102222123-380f4078db9f/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= +go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= +go.opentelemetry.io/otel v1.9.0/go.mod h1:np4EoPGzoPs3O67xUVNoPPcmSvsfOxNlNA4F4AC+0Eo= go.opentelemetry.io/otel v1.13.0 h1:1ZAKnNQKwBBxFtww/GwxNUyTf0AxkZzrukO8MeXqe4Y= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= +go.opentelemetry.io/otel/bridge/opencensus v0.25.0/go.mod h1:dkZDdaNwLlIutxK2Kc2m3jwW2M1ISaNf8/rOYVwuVHs= +go.opentelemetry.io/otel/bridge/opencensus v0.31.0/go.mod h1:hgI/HZyVIo3QL0RBmlOy2/VIQCc27OC1c/4ifwqTv4A= +go.opentelemetry.io/otel/exporters/jaeger v1.2.0/go.mod h1:KJLFbEMKTNPIfOxcg/WikIozEoKcPgJRz3Ce1vLlM8E= go.opentelemetry.io/otel/exporters/jaeger v1.13.0 h1:VAMoGujbVV8Q0JNM/cEbhzUIWWBxnEqH45HP9iBKN04= go.opentelemetry.io/otel/exporters/jaeger v1.13.0/go.mod h1:fHwbmle6mBFJA1p2ZIhilvffCdq/dM5UTIiCOmEjS+w= +go.opentelemetry.io/otel/internal/metric v0.25.0/go.mod h1:Nhuw26QSX7d6n4duoqAFi5KOQR4AuzyMcl5eXOgwxtc= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v0.25.0/go.mod h1:E884FSpQfnJOMMUaq+05IWlJ4rjZpk2s/F1Ju+TEEm8= +go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.2.0/go.mod h1:jNN8QtpvbsKhgaC6V5lHiejMoKD+V8uadoSafgHPx1U= +go.opentelemetry.io/otel/sdk v1.8.0/go.mod h1:uPSfc+yfDH2StDM/Rm35WE8gXSNdvCg023J6HeGNO0c= +go.opentelemetry.io/otel/sdk v1.9.0/go.mod h1:AEZc8nt5bd2F7BC24J5R0mrjYnpEgYHyTcM/vrSple4= go.opentelemetry.io/otel/sdk v1.13.0 h1:BHib5g8MvdqS65yo2vV1s6Le42Hm6rrw08qU6yz5JaM= go.opentelemetry.io/otel/sdk v1.13.0/go.mod h1:YLKPx5+6Vx/o1TCUYYs+bpymtkmazOMT6zoRrC7AQ7I= +go.opentelemetry.io/otel/sdk/export/metric v0.25.0/go.mod h1:Ej7NOa+WpN49EIcr1HMUYRvxXXCCnQCg2+ovdt2z8Pk= +go.opentelemetry.io/otel/sdk/metric v0.25.0/go.mod h1:G4xzj4LvC6xDDSsVXpvRVclQCbofGGg4ZU2VKKtDRfg= +go.opentelemetry.io/otel/sdk/metric v0.31.0/go.mod h1:fl0SmNnX9mN9xgU6OLYLMBMrNAsaZQi7qBwprwO3abk= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= +go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= +go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.opentelemetry.io/otel/trace v1.9.0/go.mod h1:2737Q0MuG8q1uILYm2YYVkAyLtOofiTNGg6VODnOiPo= go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.12.0/go.mod h1:X34SnWGr8Fyla9zQNO2GSO2D+TIuqB14OS8JhYocIyw= +go.uber.org/fx v1.15.0/go.mod h1:jI3RazQUhGv5KkpZIRv+kuP4CcgX3fnc0qX8bLnzbx8= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= @@ -1053,44 +2098,87 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= +go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= +golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210915214749-c084706c2272/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= -golang.org/x/exp v0.0.0-20210615023648-acb5c1269671 h1:ddvpKwqE7dm58PoWjRCmaCiA3DANEW0zWGfNYQD212Y= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20210615023648-acb5c1269671/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= +golang.org/x/exp v0.0.0-20210714144626-1041f73d31d8/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= +golang.org/x/exp v0.0.0-20210715201039-d37aa40e8013/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= +golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b h1:SCE/18RnFsLrjydh/R/s5EVvHoZprqEQUuoxK8q2Pc4= +golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1098,17 +2186,28 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1127,31 +2226,70 @@ golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201022231255-08b38378de70/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210423184538-5f58ad60dda6/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2 h1:6mzvA99KwZxbOrxww4EvWVQUnN1+xEu9tafK5ZxkYeA= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5 h1:KafLifaRFIuSJ5C+7CyFJOF9haxKNC1CEIDk8GX6X0k= +golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1164,7 +2302,10 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1175,59 +2316,121 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190524152521-dbbf3f1254d4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191025090151-53bf42e6b339/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191206220618-eeba5f6aabab/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426080607-c94f62235c83/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210511113859-b0526f3d8744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025112917-711f33c9992c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1236,67 +2439,167 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191030062658-86caa796c7ab/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200711155855-7342f9734a7d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1304,20 +2607,30 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1326,26 +2639,29 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= @@ -1363,27 +2679,161 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/cc/v3 v3.33.6/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.33.9/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.33.11/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.34.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.0/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.4/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.5/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.7/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.8/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.10/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.15/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.16/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.17/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.18/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.20/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/cc/v3 v3.35.22/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= +modernc.org/ccgo/v3 v3.9.5/go.mod h1:umuo2EP2oDSBnD3ckjaVUXMrmeAw8C8OSICVa0iFf60= +modernc.org/ccgo/v3 v3.10.0/go.mod h1:c0yBmkRFi7uW4J7fwx/JiijwOjeAeR2NoSaRVFPmjMw= +modernc.org/ccgo/v3 v3.11.0/go.mod h1:dGNposbDp9TOZ/1KBxghxtUp/bzErD0/0QW4hhSaBMI= +modernc.org/ccgo/v3 v3.11.1/go.mod h1:lWHxfsn13L3f7hgGsGlU28D9eUOf6y3ZYHKoPaKU0ag= +modernc.org/ccgo/v3 v3.11.3/go.mod h1:0oHunRBMBiXOKdaglfMlRPBALQqsfrCKXgw9okQ3GEw= +modernc.org/ccgo/v3 v3.12.4/go.mod h1:Bk+m6m2tsooJchP/Yk5ji56cClmN6R1cqc9o/YtbgBQ= +modernc.org/ccgo/v3 v3.12.6/go.mod h1:0Ji3ruvpFPpz+yu+1m0wk68pdr/LENABhTrDkMDWH6c= +modernc.org/ccgo/v3 v3.12.8/go.mod h1:Hq9keM4ZfjCDuDXxaHptpv9N24JhgBZmUG5q60iLgUo= +modernc.org/ccgo/v3 v3.12.11/go.mod h1:0jVcmyDwDKDGWbcrzQ+xwJjbhZruHtouiBEvDfoIsdg= +modernc.org/ccgo/v3 v3.12.14/go.mod h1:GhTu1k0YCpJSuWwtRAEHAol5W7g1/RRfS4/9hc9vF5I= +modernc.org/ccgo/v3 v3.12.18/go.mod h1:jvg/xVdWWmZACSgOiAhpWpwHWylbJaSzayCqNOJKIhs= +modernc.org/ccgo/v3 v3.12.20/go.mod h1:aKEdssiu7gVgSy/jjMastnv/q6wWGRbszbheXgWRHc8= +modernc.org/ccgo/v3 v3.12.21/go.mod h1:ydgg2tEprnyMn159ZO/N4pLBqpL7NOkJ88GT5zNU2dE= +modernc.org/ccgo/v3 v3.12.22/go.mod h1:nyDVFMmMWhMsgQw+5JH6B6o4MnZ+UQNw1pp52XYFPRk= +modernc.org/ccgo/v3 v3.12.25/go.mod h1:UaLyWI26TwyIT4+ZFNjkyTbsPsY3plAEB6E7L/vZV3w= +modernc.org/ccgo/v3 v3.12.29/go.mod h1:FXVjG7YLf9FetsS2OOYcwNhcdOLGt8S9bQ48+OP75cE= +modernc.org/ccgo/v3 v3.12.36/go.mod h1:uP3/Fiezp/Ga8onfvMLpREq+KUjUmYMxXPO8tETHtA8= +modernc.org/ccgo/v3 v3.12.38/go.mod h1:93O0G7baRST1vNj4wnZ49b1kLxt0xCW5Hsa2qRaZPqc= +modernc.org/ccgo/v3 v3.12.43/go.mod h1:k+DqGXd3o7W+inNujK15S5ZYuPoWYLpF5PYougCmthU= +modernc.org/ccgo/v3 v3.12.46/go.mod h1:UZe6EvMSqOxaJ4sznY7b23/k13R8XNlyWsO5bAmSgOE= +modernc.org/ccgo/v3 v3.12.47/go.mod h1:m8d6p0zNps187fhBwzY/ii6gxfjob1VxWb919Nk1HUk= +modernc.org/ccgo/v3 v3.12.50/go.mod h1:bu9YIwtg+HXQxBhsRDE+cJjQRuINuT9PUK4orOco/JI= +modernc.org/ccgo/v3 v3.12.51/go.mod h1:gaIIlx4YpmGO2bLye04/yeblmvWEmE4BBBls4aJXFiE= +modernc.org/ccgo/v3 v3.12.53/go.mod h1:8xWGGTFkdFEWBEsUmi+DBjwu/WLy3SSOrqEmKUjMeEg= +modernc.org/ccgo/v3 v3.12.54/go.mod h1:yANKFTm9llTFVX1FqNKHE0aMcQb1fuPJx6p8AcUx+74= +modernc.org/ccgo/v3 v3.12.55/go.mod h1:rsXiIyJi9psOwiBkplOaHye5L4MOOaCjHg1Fxkj7IeU= +modernc.org/ccgo/v3 v3.12.56/go.mod h1:ljeFks3faDseCkr60JMpeDb2GSO3TKAmrzm7q9YOcMU= +modernc.org/ccgo/v3 v3.12.57/go.mod h1:hNSF4DNVgBl8wYHpMvPqQWDQx8luqxDnNGCMM4NFNMc= +modernc.org/ccgo/v3 v3.12.60/go.mod h1:k/Nn0zdO1xHVWjPYVshDeWKqbRWIfif5dtsIOCUVMqM= +modernc.org/ccgo/v3 v3.12.66/go.mod h1:jUuxlCFZTUZLMV08s7B1ekHX5+LIAurKTTaugUr/EhQ= +modernc.org/ccgo/v3 v3.12.67/go.mod h1:Bll3KwKvGROizP2Xj17GEGOTrlvB1XcVaBrC90ORO84= +modernc.org/ccgo/v3 v3.12.73/go.mod h1:hngkB+nUUqzOf3iqsM48Gf1FZhY599qzVg1iX+BT3cQ= +modernc.org/ccgo/v3 v3.12.81/go.mod h1:p2A1duHoBBg1mFtYvnhAnQyI6vL0uw5PGYLSIgF6rYY= +modernc.org/ccgo/v3 v3.12.84/go.mod h1:ApbflUfa5BKadjHynCficldU1ghjen84tuM5jRynB7w= +modernc.org/ccgo/v3 v3.12.86/go.mod h1:dN7S26DLTgVSni1PVA3KxxHTcykyDurf3OgUzNqTSrU= +modernc.org/ccgo/v3 v3.12.90/go.mod h1:obhSc3CdivCRpYZmrvO88TXlW0NvoSVvdh/ccRjJYko= +modernc.org/ccgo/v3 v3.12.92/go.mod h1:5yDdN7ti9KWPi5bRVWPl8UNhpEAtCjuEE7ayQnzzqHA= +modernc.org/ccgo/v3 v3.13.1/go.mod h1:aBYVOUfIlcSnrsRVU8VRS35y2DIfpgkmVkYZ0tpIXi4= +modernc.org/ccgo/v3 v3.15.1/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0= +modernc.org/ccgo/v3 v3.15.9/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0= +modernc.org/ccgo/v3 v3.15.10/go.mod h1:wQKxoFn0ynxMuCLfFD09c8XPUCc8obfchoVR9Cn0fI8= +modernc.org/ccgo/v3 v3.15.12/go.mod h1:VFePOWoCd8uDGRJpq/zfJ29D0EVzMSyID8LCMWYbX6I= +modernc.org/ccgo/v3 v3.15.13/go.mod h1:QHtvdpeODlXjdK3tsbpyK+7U9JV4PQsrPGIbtmc0KfY= +modernc.org/ccorpus v1.11.1/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/ccorpus v1.11.4/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/golex v1.0.1/go.mod h1:QCA53QtsT1NdGkaZZkF5ezFwk4IXh4BGNafAARTC254= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/lex v1.0.0/go.mod h1:G6rxMTy3cH2iA0iXL/HRRv4Znu8MK4higxph/lE7ypk= +modernc.org/lexer v1.0.0/go.mod h1:F/Dld0YKYdZCLQ7bD0USbWL4YKCyTDRDHiDTOs0q0vk= +modernc.org/libc v1.9.8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w= +modernc.org/libc v1.9.11/go.mod h1:NyF3tsA5ArIjJ83XB0JlqhjTabTCHm9aX4XMPHyQn0Q= +modernc.org/libc v1.11.0/go.mod h1:2lOfPmj7cz+g1MrPNmX65QCzVxgNq2C5o0jdLY2gAYg= +modernc.org/libc v1.11.2/go.mod h1:ioIyrl3ETkugDO3SGZ+6EOKvlP3zSOycUETe4XM4n8M= +modernc.org/libc v1.11.5/go.mod h1:k3HDCP95A6U111Q5TmG3nAyUcp3kR5YFZTeDS9v8vSU= +modernc.org/libc v1.11.6/go.mod h1:ddqmzR6p5i4jIGK1d/EiSw97LBcE3dK24QEwCFvgNgE= +modernc.org/libc v1.11.11/go.mod h1:lXEp9QOOk4qAYOtL3BmMve99S5Owz7Qyowzvg6LiZso= +modernc.org/libc v1.11.13/go.mod h1:ZYawJWlXIzXy2Pzghaf7YfM8OKacP3eZQI81PDLFdY8= +modernc.org/libc v1.11.16/go.mod h1:+DJquzYi+DMRUtWI1YNxrlQO6TcA5+dRRiq8HWBWRC8= +modernc.org/libc v1.11.19/go.mod h1:e0dgEame6mkydy19KKaVPBeEnyJB4LGNb0bBH1EtQ3I= +modernc.org/libc v1.11.24/go.mod h1:FOSzE0UwookyT1TtCJrRkvsOrX2k38HoInhw+cSCUGk= +modernc.org/libc v1.11.26/go.mod h1:SFjnYi9OSd2W7f4ct622o/PAYqk7KHv6GS8NZULIjKY= +modernc.org/libc v1.11.27/go.mod h1:zmWm6kcFXt/jpzeCgfvUNswM0qke8qVwxqZrnddlDiE= +modernc.org/libc v1.11.28/go.mod h1:Ii4V0fTFcbq3qrv3CNn+OGHAvzqMBvC7dBNyC4vHZlg= +modernc.org/libc v1.11.31/go.mod h1:FpBncUkEAtopRNJj8aRo29qUiyx5AvAlAxzlx9GNaVM= +modernc.org/libc v1.11.34/go.mod h1:+Tzc4hnb1iaX/SKAutJmfzES6awxfU1BPvrrJO0pYLg= +modernc.org/libc v1.11.37/go.mod h1:dCQebOwoO1046yTrfUE5nX1f3YpGZQKNcITUYWlrAWo= +modernc.org/libc v1.11.39/go.mod h1:mV8lJMo2S5A31uD0k1cMu7vrJbSA3J3waQJxpV4iqx8= +modernc.org/libc v1.11.42/go.mod h1:yzrLDU+sSjLE+D4bIhS7q1L5UwXDOw99PLSX0BlZvSQ= +modernc.org/libc v1.11.44/go.mod h1:KFq33jsma7F5WXiYelU8quMJasCCTnHK0mkri4yPHgA= +modernc.org/libc v1.11.45/go.mod h1:Y192orvfVQQYFzCNsn+Xt0Hxt4DiO4USpLNXBlXg/tM= +modernc.org/libc v1.11.47/go.mod h1:tPkE4PzCTW27E6AIKIR5IwHAQKCAtudEIeAV1/SiyBg= +modernc.org/libc v1.11.49/go.mod h1:9JrJuK5WTtoTWIFQ7QjX2Mb/bagYdZdscI3xrvHbXjE= +modernc.org/libc v1.11.51/go.mod h1:R9I8u9TS+meaWLdbfQhq2kFknTW0O3aw3kEMqDDxMaM= +modernc.org/libc v1.11.53/go.mod h1:5ip5vWYPAoMulkQ5XlSJTy12Sz5U6blOQiYasilVPsU= +modernc.org/libc v1.11.54/go.mod h1:S/FVnskbzVUrjfBqlGFIPA5m7UwB3n9fojHhCNfSsnw= +modernc.org/libc v1.11.55/go.mod h1:j2A5YBRm6HjNkoSs/fzZrSxCuwWqcMYTDPLNx0URn3M= +modernc.org/libc v1.11.56/go.mod h1:pakHkg5JdMLt2OgRadpPOTnyRXm/uzu+Yyg/LSLdi18= +modernc.org/libc v1.11.58/go.mod h1:ns94Rxv0OWyoQrDqMFfWwka2BcaF6/61CqJRK9LP7S8= +modernc.org/libc v1.11.71/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw= +modernc.org/libc v1.11.75/go.mod h1:dGRVugT6edz361wmD9gk6ax1AbDSe0x5vji0dGJiPT0= +modernc.org/libc v1.11.82/go.mod h1:NF+Ek1BOl2jeC7lw3a7Jj5PWyHPwWD4aq3wVKxqV1fI= +modernc.org/libc v1.11.86/go.mod h1:ePuYgoQLmvxdNT06RpGnaDKJmDNEkV7ZPKI2jnsvZoE= +modernc.org/libc v1.11.87/go.mod h1:Qvd5iXTeLhI5PS0XSyqMY99282y+3euapQFxM7jYnpY= +modernc.org/libc v1.11.88/go.mod h1:h3oIVe8dxmTcchcFuCcJ4nAWaoiwzKCdv82MM0oiIdQ= +modernc.org/libc v1.11.98/go.mod h1:ynK5sbjsU77AP+nn61+k+wxUGRx9rOFcIqWYYMaDZ4c= +modernc.org/libc v1.11.101/go.mod h1:wLLYgEiY2D17NbBOEp+mIJJJBGSiy7fLL4ZrGGZ+8jI= +modernc.org/libc v1.12.0/go.mod h1:2MH3DaF/gCU8i/UBiVE1VFRos4o523M7zipmwH8SIgQ= +modernc.org/libc v1.14.1/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk= +modernc.org/libc v1.14.2/go.mod h1:MX1GBLnRLNdvmK9azU9LCxZ5lMyhrbEMK8rG3X/Fe34= +modernc.org/libc v1.14.3/go.mod h1:GPIvQVOVPizzlqyRX3l756/3ppsAgg1QgPxjr5Q4agQ= +modernc.org/libc v1.14.5/go.mod h1:2PJHINagVxO4QW/5OQdRrvMYo+bm5ClpUFfyXCYl9ak= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc= +modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.14.6/go.mod h1:yiCvMv3HblGmzENNIaNtFhfaNIwcla4u2JQEwJPzfEc= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/tcl v1.11.0/go.mod h1:zsTUpbQ+NxQEjOjCUlImDLPv1sG8Ww0qp66ZvyOxCgw= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +modernc.org/z v1.3.0/go.mod h1:+mvgLH814oDjtATDdT3rs84JnUIpkvAF5B8AVkNlE2g= +pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/extern/boostd-data/ldb/db.go b/extern/boostd-data/ldb/db.go index 48b3cb524..1631d60f5 100644 --- a/extern/boostd-data/ldb/db.go +++ b/extern/boostd-data/ldb/db.go @@ -4,9 +4,12 @@ import ( "context" "encoding/binary" "encoding/json" + "errors" "fmt" + "time" - "github.com/filecoin-project/boost/cmd/boostd-data/model" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" ds "github.com/ipfs/go-datastore" @@ -33,21 +36,31 @@ var ( prefixMhtoPieceCids uint64 = 2 sprefixMhtoPieceCids string - size = binary.MaxVarintLen64 + // LevelDB key prefix for Flagged pieces table. + // LevelDB keys will be built by concatenating PieceCid to this prefix. + prefixPieceCidToFlagged uint64 = 3 + sprefixPieceCidToFlagged string + + ///////////////////////////////////////// + // Prefixes up to 100 are system prefixes ) func init() { - buf := make([]byte, size) + buf := make([]byte, binary.MaxVarintLen64) binary.PutUvarint(buf, keyNextCursor) dskeyNextCursor = datastore.NewKey(string(buf)) - buf = make([]byte, size) + buf = make([]byte, binary.MaxVarintLen64) binary.PutUvarint(buf, prefixPieceCidToCursor) sprefixPieceCidToCursor = string(buf) - buf = make([]byte, size) + buf = make([]byte, binary.MaxVarintLen64) binary.PutUvarint(buf, prefixMhtoPieceCids) sprefixMhtoPieceCids = string(buf) + + buf = make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(buf, prefixPieceCidToFlagged) + sprefixPieceCidToFlagged = string(buf) } type DB struct { @@ -63,12 +76,31 @@ func newDB(path string, readonly bool) (*DB, error) { CompactionTableSize: 4 * opt.MiB, }) if err != nil { - return nil, err + return nil, fmt.Errorf("creating level db datstore: %w", err) } return &DB{ldb}, nil } +func (db *DB) InitCursor(ctx context.Context) error { + _, err := db.Get(ctx, dskeyNextCursor) + if err == nil { + // Cursor has already been initialized, so just return + log.Debug("leveldb cursor already initialized") + return nil + } + + if errors.Is(err, ds.ErrNotFound) { + // Cursor has not yet been initialized so initialize it + log.Debug("initializing leveldb cursor") + err = db.SetNextCursor(ctx, 100) + if err == nil { + return nil + } + } + return fmt.Errorf("initializing database cursor: %w", err) +} + // NextCursor func (db *DB) NextCursor(ctx context.Context) (uint64, string, error) { b, err := db.Get(ctx, dskeyNextCursor) @@ -82,7 +114,7 @@ func (db *DB) NextCursor(ctx context.Context) (uint64, string, error) { // SetNextCursor func (db *DB) SetNextCursor(ctx context.Context, cursor uint64) error { - buf := make([]byte, size) + buf := make([]byte, binary.MaxVarintLen64) binary.PutUvarint(buf, cursor) return db.Put(ctx, dskeyNextCursor, buf) @@ -90,6 +122,9 @@ func (db *DB) SetNextCursor(ctx context.Context, cursor uint64) error { // GetPieceCidsByMultihash func (db *DB) GetPieceCidsByMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.get_piece_cids_by_multihash") + defer span.End() + key := datastore.NewKey(fmt.Sprintf("%s%s", sprefixMhtoPieceCids, mh.String())) val, err := db.Get(ctx, key) @@ -107,6 +142,9 @@ func (db *DB) GetPieceCidsByMultihash(ctx context.Context, mh multihash.Multihas // SetMultihashToPieceCid func (db *DB) SetMultihashesToPieceCid(ctx context.Context, recs []carindex.Record, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "db.set_multihashes_to_piece_cid") + defer span.End() + batch, err := db.Batch(ctx) if err != nil { return fmt.Errorf("failed to create ds batch: %w", err) @@ -178,42 +216,125 @@ func (db *DB) SetMultihashesToPieceCid(ctx context.Context, recs []carindex.Reco return nil } +// SetPieceCidToFlagged +func (db *DB) SetPieceCidToFlagged(ctx context.Context, pieceCid cid.Cid, fm LeveldbFlaggedMetadata) error { + ctx, span := tracing.Tracer.Start(ctx, "db.set_piece_cid_to_flagged") + defer span.End() + + b, err := json.Marshal(fm) + if err != nil { + return err + } + + key := datastore.NewKey(fmt.Sprintf("%s/%s", sprefixPieceCidToFlagged, pieceCid.String())) + + return db.Put(ctx, key, b) +} + +// GetPieceCidToFlagged +func (db *DB) GetPieceCidToFlagged(ctx context.Context, pieceCid cid.Cid) (LeveldbFlaggedMetadata, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.get_piece_cid_to_flagged") + defer span.End() + + var metadata LeveldbFlaggedMetadata + + key := datastore.NewKey(fmt.Sprintf("%s/%s", sprefixPieceCidToFlagged, pieceCid.String())) + + b, err := db.Get(ctx, key) + if err != nil { + return metadata, fmt.Errorf("getting flagged metadata for piece %s: %w", pieceCid, err) + } + + err = json.Unmarshal(b, &metadata) + if err != nil { + return metadata, fmt.Errorf("unmarshaling flagged metadata for piece %s: %w", pieceCid, err) + } + + return metadata, nil +} + // SetPieceCidToMetadata -func (db *DB) SetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid, md model.Metadata) error { +func (db *DB) SetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid, md LeveldbMetadata) error { + ctx, span := tracing.Tracer.Start(ctx, "db.set_piece_cid_to_metadata") + defer span.End() + b, err := json.Marshal(md) if err != nil { return err } - key := datastore.NewKey(fmt.Sprintf("%s%s", sprefixPieceCidToCursor, pieceCid.String())) + key := datastore.NewKey(fmt.Sprintf("%s/%s", sprefixPieceCidToCursor, pieceCid.String())) return db.Put(ctx, key, b) } // GetPieceCidToMetadata -func (db *DB) GetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { - var metadata model.Metadata +func (db *DB) GetPieceCidToMetadata(ctx context.Context, pieceCid cid.Cid) (LeveldbMetadata, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.get_piece_cid_to_metadata") + defer span.End() - key := datastore.NewKey(fmt.Sprintf("%s%s", sprefixPieceCidToCursor, pieceCid.String())) + var metadata LeveldbMetadata + + key := datastore.NewKey(fmt.Sprintf("%s/%s", sprefixPieceCidToCursor, pieceCid.String())) b, err := db.Get(ctx, key) if err != nil { - return metadata, err + return metadata, fmt.Errorf("getting piece metadata for piece %s: %w", pieceCid, err) } err = json.Unmarshal(b, &metadata) if err != nil { - return metadata, err + return metadata, fmt.Errorf("unmarshaling piece metadata for piece %s: %w", pieceCid, err) } return metadata, nil } +func (db *DB) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { + ctx, span := tracing.Tracer.Start(ctx, "db.set_car_size") + defer span.End() + + md, err := db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + return fmt.Errorf("getting piece metadata for piece %s: %w", pieceCid, err) + } + + // Set the car size on each deal (should be the same for all deals) + for _, dl := range md.Deals { + dl.CarLength = size + } + + return db.SetPieceCidToMetadata(ctx, pieceCid, md) +} + +func (db *DB) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err error) error { + ctx, span := tracing.Tracer.Start(ctx, "db.mark_piece_index_errored") + defer span.End() + + md, err := db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + return fmt.Errorf("getting piece metadata for piece %s: %w", pieceCid, err) + } + + if md.Error != "" { + // If the error state has already been set, don't over-write the existing error + return nil + } + + md.Error = err.Error() + md.ErrorType = fmt.Sprintf("%T", err) + + return db.SetPieceCidToMetadata(ctx, pieceCid, md) +} + // AllRecords func (db *DB) AllRecords(ctx context.Context, cursor uint64) ([]model.Record, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.all_records") + defer span.End() + var records []model.Record - buf := make([]byte, size) + buf := make([]byte, binary.MaxVarintLen64) binary.PutUvarint(buf, cursor) var q query.Query @@ -238,38 +359,160 @@ func (db *DB) AllRecords(ctx context.Context, cursor uint64) ([]model.Record, er kcid := cid.NewCidV1(cid.Raw, m) - offset, _ := binary.Uvarint(r.Value) + offset, n := binary.Uvarint(r.Value) + size, n := binary.Uvarint(r.Value[n:]) records = append(records, model.Record{ - Cid: kcid, - Offset: offset, + Cid: kcid, + OffsetSize: model.OffsetSize{ + Offset: offset, + Size: size, + }, }) } return records, nil } -// AddOffset -func (db *DB) AddOffset(ctx context.Context, cursorPrefix string, m multihash.Multihash, offset uint64) error { - key := datastore.NewKey(fmt.Sprintf("%s%s", cursorPrefix, m.String())) +// AddIndexRecord +func (db *DB) AddIndexRecord(ctx context.Context, cursorPrefix string, rec model.Record) error { + ctx, span := tracing.Tracer.Start(ctx, "db.add_index_record") + defer span.End() + + key := datastore.NewKey(fmt.Sprintf("%s%s", cursorPrefix, rec.Cid.Hash().String())) - value := make([]byte, size) - binary.PutUvarint(value, offset) + value := make([]byte, 2*binary.MaxVarintLen64) + no := binary.PutUvarint(value, rec.Offset) + ns := binary.PutUvarint(value[no:], rec.Size) - return db.Put(ctx, key, value) + return db.Put(ctx, key, value[:no+ns]) } -// GetOffset -func (db *DB) GetOffset(ctx context.Context, cursorPrefix string, m multihash.Multihash) (uint64, error) { +// GetOffsetSize +func (db *DB) GetOffsetSize(ctx context.Context, cursorPrefix string, m multihash.Multihash) (*model.OffsetSize, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.get_offset") + defer span.End() + key := datastore.NewKey(fmt.Sprintf("%s%s", cursorPrefix, m.String())) b, err := db.Get(ctx, key) if err != nil { - return 0, err + return nil, err + } + + offset, n := binary.Uvarint(b) + size, n := binary.Uvarint(b[n:]) + return &model.OffsetSize{ + Offset: offset, + Size: size, + }, nil +} + +var ( + // The minimum frequency with which to check pieces for errors (eg bad index) + MinPieceCheckPeriod = 30 * time.Second + + // in-memory cursor to the position we reached in the leveldb table with respect to piece cids to process for errors with the doctor + offset int + + // checked keeps track in memory when was the last time we processed a given piece cid + checked map[string]time.Time + + // batch limit for each NextPiecesToCheck call + PiecesToTrackerBatchSize = 1024 +) + +func init() { + checked = make(map[string]time.Time) +} + +func (db *DB) NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.next_pieces_to_check") + defer span.End() + + q := query.Query{ + Prefix: "/" + sprefixPieceCidToCursor + "/", + KeysOnly: true, + Limit: PiecesToTrackerBatchSize, + Offset: offset, + } + results, err := db.Query(ctx, q) + if err != nil { + return nil, fmt.Errorf("listing pieces in database: %w", err) } - offset, _ := binary.Uvarint(b) - return offset, nil + var pieceCids []cid.Cid + + now := time.Now() + + var i int + for { + r, ok := results.NextSync() + if !ok { + break + } + i++ + + k := r.Key[len(q.Prefix):] + if t, ok := checked[k]; ok { + alreadyChecked := t.After(now.Add(-MinPieceCheckPeriod)) + + if alreadyChecked { + continue + } + } + checked[k] = now + + pieceCid, err := cid.Parse(k) + if err != nil { + return nil, fmt.Errorf("parsing piece cid '%s': %w", k, err) + } + + pieceCids = append(pieceCids, pieceCid) + } + offset += i + + // if we got less pieces than the specified limit, we must be at the end of the table, + // so reset the cursor + if i < PiecesToTrackerBatchSize-1 { + offset = 0 + } + + log.Debugw("NextPiecesToCheck: returning piececids", "len", len(pieceCids), "offset", offset) + + return pieceCids, nil +} + +func (db *DB) ListPieces(ctx context.Context) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.list_pieces") + defer span.End() + + q := query.Query{ + Prefix: "/" + sprefixPieceCidToCursor + "/", + KeysOnly: true, + } + results, err := db.Query(ctx, q) + if err != nil { + return nil, fmt.Errorf("listing pieces in database: %w", err) + } + + var pieceCids []cid.Cid + for { + r, ok := results.NextSync() + if !ok { + break + } + + k := r.Key[len(q.Prefix):] + pieceCid, err := cid.Parse(k) + if err != nil { + return nil, fmt.Errorf("parsing piece cid '%s': %w", k, err) + } + + pieceCids = append(pieceCids, pieceCid) + } + + return pieceCids, nil } func has(list []cid.Cid, v cid.Cid) bool { @@ -280,3 +523,206 @@ func has(list []cid.Cid, v cid.Cid) bool { } return false } + +// RemoveMetadata +func (db *DB) RemovePieceMetadata(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "db.remove_piece_metadata") + defer span.End() + + var metadata LeveldbMetadata + + key := datastore.NewKey(fmt.Sprintf("%s/%s", sprefixPieceCidToCursor, pieceCid.String())) + + piece, err := db.Get(ctx, key) + if err != nil { + return err + } + + err = json.Unmarshal(piece, &metadata) + if err != nil { + return fmt.Errorf("error while reading metadata: %w", err) + } + + // Remove all multihashes before, as without Metadata, they are useless + // This order is important as metadata.Cursor is required in case RemoveAllRecords fails + // and needs to be run manually + if err = db.RemoveIndexes(ctx, metadata.Cursor, pieceCid); err != nil { + return err + } + + // TODO: Requires DB compaction for removing the key + if err = db.Delete(ctx, key); err != nil { + return err + } + + return nil +} + +// RemoveIndexes +// It removes multihash -> pieceCid and if empty record is left then multihash -> offset +// entry is also removed +func (db *DB) RemoveIndexes(ctx context.Context, cursor uint64, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "db.remove_indexes") + defer span.End() + + buf := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(buf, cursor) + + var q query.Query + q.Prefix = fmt.Sprintf("%d/", cursor) + results, err := db.Query(ctx, q) + if err != nil { + return fmt.Errorf("error querying the database: %w", err) + } + + batch, err := db.Batch(ctx) + if err != nil { + return fmt.Errorf("error in creating batching: %w", err) + } + + for { + r, ok := results.NextSync() + if !ok { + break + } + + m := r.Key[len(q.Prefix)+1:] + + err = func() error { + key := datastore.NewKey(fmt.Sprintf("%s%s", sprefixMhtoPieceCids, m)) + + val, err := db.Get(ctx, key) + if err != nil && err != ds.ErrNotFound { + return fmt.Errorf("failed to get value for multihash %s, err: %w", m, err) + } + + if errors.Is(err, ds.ErrNotFound) { + return nil + } + + var pcids []cid.Cid + if err := json.Unmarshal(val, &pcids); err != nil { + return fmt.Errorf("failed to unmarshal pieceCids slice: %w", err) + } + + if !has(pcids, pieceCid) { + return nil + } + + if len(pcids) <= 1 { + // Remove multihash -> pieceCId (key+value) + if err := batch.Delete(ctx, key); err != nil { + return fmt.Errorf("failed to batch delete multihash to pieceCid mh=%s, pieceCid=%s err%w", key, pcids[0], err) + } + return nil + } + + // Remove multihash -> pieceCId (value only) + for i, v := range pcids { + if v == pieceCid { + pcids[i] = pcids[len(pcids)-1] + pcids = pcids[:len(pcids)-1] + } + } + + b, err := json.Marshal(pcids) + if err != nil { + return fmt.Errorf("failed to marshal pieceCids slice: %w", err) + } + if err := batch.Put(ctx, key, b); err != nil { + return fmt.Errorf("failed to batch put mh=%s, err%w", m, err) + } + + return nil + }() + if err != nil { + return err + } + + // Remove (cursor+multihash) -> Offset + if err := batch.Delete(ctx, ds.NewKey(r.Key)); err != nil { + return fmt.Errorf("failed to batch delete mh=%s, err%w", r.Key, err) + } + } + + if err := batch.Commit(ctx); err != nil { + return fmt.Errorf("failed to commit batch: %w", err) + } + + return nil +} + +func (db *DB) ListFlaggedPieces(ctx context.Context) ([]model.FlaggedPiece, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.list_flagged_pieces") + defer span.End() + + q := query.Query{ + Prefix: "/" + sprefixPieceCidToFlagged + "/", + KeysOnly: false, + } + results, err := db.Query(ctx, q) + if err != nil { + return nil, fmt.Errorf("listing flagged pieces in database: %w", err) + } + + var records []model.FlaggedPiece + for { + r, ok := results.NextSync() + if !ok { + break + } + + k := r.Key[len(q.Prefix):] + pieceCid, err := cid.Parse(k) + if err != nil { + return nil, fmt.Errorf("parsing piece cid '%s': %w", k, err) + } + + var v LeveldbFlaggedMetadata + err = json.Unmarshal(r.Value, &v) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal LeveldbFlaggedMetadata: %w; %v", err, r.Value) + } + + records = append(records, model.FlaggedPiece{CreatedAt: v.CreatedAt, PieceCid: pieceCid}) + } + + return records, nil +} + +func (db *DB) FlaggedPiecesCount(ctx context.Context) (int, error) { + ctx, span := tracing.Tracer.Start(ctx, "db.flagged_pieces_count") + defer span.End() + + q := query.Query{ + Prefix: "/" + sprefixPieceCidToFlagged + "/", + KeysOnly: true, + } + results, err := db.Query(ctx, q) + if err != nil { + return -1, fmt.Errorf("listing flagged pieces in database: %w", err) + } + + var i int + for { + _, ok := results.NextSync() + if !ok { + break + } + + i++ + } + + return i, nil +} + +// DeletePieceCidToFlagged +func (db *DB) DeletePieceCidToFlagged(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "db.delete_piece_flagged_metadata") + defer span.End() + + key := datastore.NewKey(fmt.Sprintf("%s/%s", sprefixPieceCidToFlagged, pieceCid.String())) + + // TODO: Requires DB compaction for removing the key + return db.Delete(ctx, key) +} diff --git a/extern/boostd-data/ldb/service.go b/extern/boostd-data/ldb/service.go index 4b13805d4..80efadb2f 100644 --- a/extern/boostd-data/ldb/service.go +++ b/extern/boostd-data/ldb/service.go @@ -8,58 +8,84 @@ import ( "sync" "time" - "github.com/filecoin-project/boost/cmd/boostd-data/model" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/tracing" + "github.com/filecoin-project/boostd-data/svc/types" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" ds "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" - "github.com/ipld/go-car/v2/index" carindex "github.com/ipld/go-car/v2/index" - "github.com/multiformats/go-multicodec" - "github.com/multiformats/go-multihash" mh "github.com/multiformats/go-multihash" ) +// The current piece metadata version. This version will be used when doing +// data migrations (migrations are not yet implemented in version 1). +const pieceMetadataVersion = "1" + var log = logging.Logger("boostd-data-ldb") +type LeveldbFlaggedMetadata struct { + CreatedAt time.Time `json:"c"` + UpdatedAt time.Time `json:"u"` +} + +type LeveldbMetadata struct { + model.Metadata + Cursor uint64 `json:"c"` +} + +func newLeveldbMetadata() LeveldbMetadata { + return LeveldbMetadata{ + Metadata: model.Metadata{Version: pieceMetadataVersion}, + } +} + type Store struct { sync.Mutex - db *DB + db *DB + repopath string +} + +var _ types.ServiceImpl = (*Store)(nil) + +func NewStore(repopath string) *Store { + return &Store{repopath: repopath} } -func NewStore(_repopath string) *Store { - // tests - repopath := _repopath - if _repopath == "" { +func (s *Store) Start(ctx context.Context) error { + repopath := s.repopath + if repopath == "" { + // used by tests var err error repopath, err = ioutil.TempDir("", "ds-leveldb") if err != nil { - panic(err) + return fmt.Errorf("creating leveldb tmp dir: %w", err) } } - db, err := newDB(repopath, false) + var err error + s.db, err = newDB(repopath, false) if err != nil { - panic(err) + return err } - // tests - if _repopath == "" { - // prepare db - log.Debug("preparing db with next cursor") - db.SetNextCursor(context.Background(), 100) + // Prepare db with a cursor + err = s.db.InitCursor(ctx) + if err != nil { + return err } - log.Debugw("new piece meta service", "repo path", repopath) - - return &Store{ - db: db, - } + log.Debugw("new leveldb local index directory service", "repo path", repopath) + return nil } -func (s *Store) AddDealForPiece(pieceCid cid.Cid, dealInfo model.DealInfo) error { +func (s *Store) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { log.Debugw("handle.add-deal-for-piece", "piece-cid", pieceCid) + ctx, span := tracing.Tracer.Start(ctx, "store.add_deal_for_piece") + defer span.End() + defer func(now time.Time) { log.Debugw("handled.add-deal-for-piece", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) @@ -67,15 +93,27 @@ func (s *Store) AddDealForPiece(pieceCid cid.Cid, dealInfo model.DealInfo) error s.Lock() defer s.Unlock() - ctx := context.Background() - + // Get the existing deals for the piece md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) if err != nil { - return err + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("getting piece cid metadata for piece %s: %w", pieceCid, err) + } + // there isn't yet any metadata, so create new metadata + md = newLeveldbMetadata() } + // Check if the deal has already been added + for _, dl := range md.Deals { + if dl == dealInfo { + return nil + } + } + + // Add the deal to the list md.Deals = append(md.Deals, dealInfo) + // Write the piece metadata back to the db err = s.db.SetPieceCidToMetadata(ctx, pieceCid, md) if err != nil { return err @@ -84,54 +122,90 @@ func (s *Store) AddDealForPiece(pieceCid cid.Cid, dealInfo model.DealInfo) error return nil } -func (s *Store) GetRecords(pieceCid cid.Cid) ([]model.Record, error) { - log.Debugw("handle.get-iterable-index", "piece-cid", pieceCid) +func (s *Store) SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error { + log.Debugw("handle.set-car-size", "piece-cid", pieceCid, "size", size) + + ctx, span := tracing.Tracer.Start(context.Background(), "store.set-car-size") + defer span.End() defer func(now time.Time) { - log.Debugw("handled.get-iterable-index", "took", fmt.Sprintf("%s", time.Since(now))) + log.Debugw("handled.set-car-size", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) s.Lock() defer s.Unlock() - ctx := context.Background() + err := s.db.SetCarSize(ctx, pieceCid, size) + return normalizePieceCidError(pieceCid, err) +} - md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) - if err != nil { - return nil, err - } +func (s *Store) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, idxErr string) error { + log.Debugw("handle.mark-piece-index-errored", "piece-cid", pieceCid, "err", idxErr) - records, err := s.db.AllRecords(ctx, md.Cursor) + ctx, span := tracing.Tracer.Start(ctx, "store.mark-piece-index-errored") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.mark-piece-index-errored", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) + + s.Lock() + defer s.Unlock() + + err := s.db.MarkIndexErrored(ctx, pieceCid, errors.New(idxErr)) if err != nil { - return nil, err + return normalizePieceCidError(pieceCid, err) } - return records, nil + return s.FlagPiece(ctx, pieceCid) } -func (s *Store) GetOffset(pieceCid cid.Cid, hash mh.Multihash) (uint64, error) { - log.Debugw("handle.get-offset", "piece-cid", pieceCid) +func (s *Store) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { + log.Debugw("handle.get-offset-size", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.get_offset_size") + defer span.End() defer func(now time.Time) { - log.Debugw("handled.get-offset", "took", fmt.Sprintf("%s", time.Since(now))) + log.Debugw("handled.get-offset-size", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) s.Lock() defer s.Unlock() - ctx := context.Background() + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + return nil, normalizePieceCidError(pieceCid, err) + } + + return s.db.GetOffsetSize(ctx, fmt.Sprintf("%d", md.Cursor)+"/", hash) +} + +func (s *Store) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) { + log.Debugw("handle.get-piece-metadata", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.get_piece_metadata") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.get-piece-metadata", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) if err != nil { - return 0, err + err = normalizePieceCidError(pieceCid, err) + return model.Metadata{}, fmt.Errorf("getting piece metadata for piece %s: %w", pieceCid, err) } - return s.db.GetOffset(ctx, fmt.Sprintf("%d", md.Cursor)+"/", hash) + return md.Metadata, nil } -func (s *Store) GetPieceDeals(pieceCid cid.Cid) ([]model.DealInfo, error) { +func (s *Store) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) { log.Debugw("handle.get-piece-deals", "piece-cid", pieceCid) + ctx, span := tracing.Tracer.Start(ctx, "store.get_piece_deals") + defer span.End() + defer func(now time.Time) { log.Debugw("handled.get-piece-deals", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) @@ -139,20 +213,22 @@ func (s *Store) GetPieceDeals(pieceCid cid.Cid) ([]model.DealInfo, error) { s.Lock() defer s.Unlock() - ctx := context.Background() - md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) if err != nil { - return nil, err + err = normalizePieceCidError(pieceCid, err) + return nil, fmt.Errorf("getting piece deals for piece %s: %w", pieceCid, err) } return md.Deals, nil } // Get all pieces that contain a multihash (used when retrieving by payload CID) -func (s *Store) PiecesContainingMultihash(m mh.Multihash) ([]cid.Cid, error) { +func (s *Store) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ([]cid.Cid, error) { log.Debugw("handle.pieces-containing-mh", "mh", m) + ctx, span := tracing.Tracer.Start(ctx, "store.pieces_containing_multihash") + defer span.End() + defer func(now time.Time) { log.Debugw("handled.pieces-containing-mh", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) @@ -160,13 +236,16 @@ func (s *Store) PiecesContainingMultihash(m mh.Multihash) ([]cid.Cid, error) { s.Lock() defer s.Unlock() - ctx := context.Background() - return s.db.GetPieceCidsByMultihash(ctx, m) + pcs, err := s.db.GetPieceCidsByMultihash(ctx, m) + return pcs, normalizeMultihashError(m, err) } -func (s *Store) GetIndex(pieceCid cid.Cid) ([]model.Record, error) { +func (s *Store) GetIndex(ctx context.Context, pieceCid cid.Cid) ([]model.Record, error) { log.Warnw("handle.get-index", "pieceCid", pieceCid) + ctx, span := tracing.Tracer.Start(ctx, "store.get_index") + defer span.End() + defer func(now time.Time) { log.Warnw("handled.get-index", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) @@ -174,16 +253,15 @@ func (s *Store) GetIndex(pieceCid cid.Cid) ([]model.Record, error) { s.Lock() defer s.Unlock() - ctx := context.Background() - md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) if err != nil { - return nil, err + return nil, normalizePieceCidError(pieceCid, err) } records, err := s.db.AllRecords(ctx, md.Cursor) if err != nil { - return nil, err + err = normalizePieceCidError(pieceCid, err) + return nil, fmt.Errorf("getting all records for cursor %d: %w", md.Cursor, err) } log.Warnw("handle.get-index.records", "len(records)", len(records)) @@ -191,9 +269,38 @@ func (s *Store) GetIndex(pieceCid cid.Cid) ([]model.Record, error) { return records, nil } -func (s *Store) AddIndex(pieceCid cid.Cid, records []model.Record) error { +func (s *Store) IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) { + t, err := s.IndexedAt(ctx, pieceCid) + if err != nil { + return false, err + } + return !t.IsZero(), nil +} + +func (s *Store) IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, error) { + log.Debugw("handle.is-complete-index", "pieceCid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.is_incomplete_index") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.is-complete-index", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) + + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + return false, normalizePieceCidError(pieceCid, err) + } + + return md.CompleteIndex, nil +} + +func (s *Store) AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error { log.Debugw("handle.add-index", "records", len(records)) + ctx, span := tracing.Tracer.Start(ctx, "store.add_index") + defer span.End() + defer func(now time.Time) { log.Debugw("handled.add-index", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) @@ -201,8 +308,6 @@ func (s *Store) AddIndex(pieceCid cid.Cid, records []model.Record) error { s.Lock() defer s.Unlock() - ctx := context.Background() - var recs []carindex.Record for _, r := range records { recs = append(recs, carindex.Record{ @@ -222,42 +327,37 @@ func (s *Store) AddIndex(pieceCid cid.Cid, records []model.Record) error { return fmt.Errorf("couldnt generate next cursor: %w", err) } - // alloacte metadata for pieceCid + // allocate metadata for pieceCid err = s.db.SetNextCursor(ctx, cursor+1) if err != nil { return err } - mis := make(index.MultihashIndexSorted) - err = mis.Load(recs) - if err != nil { - return err - } - - var subject index.Index - subject = &mis - // process index and store entries - switch idx := subject.(type) { - case index.IterableIndex: - err := idx.ForEach(func(m multihash.Multihash, offset uint64) error { - - return s.db.AddOffset(ctx, keyCursorPrefix, m, offset) - }) + for _, rec := range records { + err := s.db.AddIndexRecord(ctx, keyCursorPrefix, rec) if err != nil { return err } - - default: - return errors.New(fmt.Sprintf("wanted %v but got %v\n", multicodec.CarMultihashIndexSorted, idx.Codec())) } - // mark that indexing is complete - md := model.Metadata{ - Cursor: cursor, - IndexedAt: time.Now(), + // get the metadata for the piece + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("getting piece cid metadata for piece %s: %w", pieceCid, err) + } + // there isn't yet any metadata, so create new metadata + md = newLeveldbMetadata() } + // mark indexing as complete + md.Cursor = cursor + md.IndexedAt = time.Now() + md.CompleteIndex = isCompleteIndex + md.Error = "" + md.ErrorType = "" + err = s.db.SetPieceCidToMetadata(ctx, pieceCid, md) if err != nil { return err @@ -271,9 +371,12 @@ func (s *Store) AddIndex(pieceCid cid.Cid, records []model.Record) error { return nil } -func (s *Store) IndexedAt(pieceCid cid.Cid) (time.Time, error) { +func (s *Store) IndexedAt(ctx context.Context, pieceCid cid.Cid) (time.Time, error) { log.Debugw("handle.indexed-at", "pieceCid", pieceCid) + ctx, span := tracing.Tracer.Start(ctx, "store.indexed_at") + defer span.End() + defer func(now time.Time) { log.Debugw("handled.indexed-at", "took", fmt.Sprintf("%s", time.Since(now))) }(time.Now()) @@ -281,12 +384,235 @@ func (s *Store) IndexedAt(pieceCid cid.Cid) (time.Time, error) { s.Lock() defer s.Unlock() - ctx := context.Background() - md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) - if err != nil && err != ds.ErrNotFound { + if err != nil && !errors.Is(err, ds.ErrNotFound) { return time.Time{}, err } return md.IndexedAt, nil } + +func (s *Store) ListPieces(ctx context.Context) ([]cid.Cid, error) { + log.Debugw("handle.list-pieces") + + ctx, span := tracing.Tracer.Start(ctx, "store.list_pieces") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.list-pieces", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) + + return s.db.ListPieces(ctx) +} + +func (s *Store) NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "store.next_pieces_to_check") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.next-pieces-to-check", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) + + return s.db.NextPiecesToCheck(ctx) +} + +func (s *Store) FlagPiece(ctx context.Context, pieceCid cid.Cid) error { + log.Debugw("handle.flag-piece", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.flag_piece") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.flag-piece", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) + + s.Lock() + defer s.Unlock() + + now := time.Now() + + // Get the existing deals for the piece + fm, err := s.db.GetPieceCidToFlagged(ctx, pieceCid) + if err != nil { + if !errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("getting piece cid flagged metadata for piece %s: %w", pieceCid, err) + } + // there isn't yet any flagged metadata, so create new metadata + fm = LeveldbFlaggedMetadata{CreatedAt: now} + } + + fm.UpdatedAt = now + + // Write the piece metadata back to the db + err = s.db.SetPieceCidToFlagged(ctx, pieceCid, fm) + if err != nil { + return err + } + + return nil +} + +func (s *Store) UnflagPiece(ctx context.Context, pieceCid cid.Cid) error { + log.Debugw("handle.unflag-piece", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.unflag_piece") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.unflag-piece", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) + + s.Lock() + defer s.Unlock() + + err := s.db.DeletePieceCidToFlagged(ctx, pieceCid) + if err != nil { + return fmt.Errorf("deleting piece cid flagged metadata for piece %s: %w", pieceCid, err) + } + return nil +} + +func (s *Store) FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) { + log.Debugw("handle.flagged-pieces-list") + + ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces_list") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.flagged-pieces-list", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) + + return s.db.ListFlaggedPieces(ctx) +} + +func (s *Store) FlaggedPiecesCount(ctx context.Context) (int, error) { + log.Debugw("handle.flagged-pieces-count") + + ctx, span := tracing.Tracer.Start(ctx, "store.flagged_pieces_count") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.flagged-pieces-count", "took", fmt.Sprintf("%s", time.Since(now))) + }(time.Now()) + + return s.db.FlaggedPiecesCount(ctx) +} + +func normalizePieceCidError(pieceCid cid.Cid, err error) error { + if err == nil { + return nil + } + if errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("piece %s: %s", pieceCid, types.ErrNotFound) + } + return err +} + +func normalizeMultihashError(m mh.Multihash, err error) error { + if err == nil { + return nil + } + if errors.Is(err, ds.ErrNotFound) { + return fmt.Errorf("multihash %s: %s", m, types.ErrNotFound) + } + return err +} + +// RemoveDealForPiece remove Single deal for pieceCID. If []Deals is empty then Metadata is removed as well +func (s *Store) RemoveDealForPiece(ctx context.Context, pieceCid cid.Cid, dealUuid string) error { + log.Debugw("handle.remove-deal-for-piece", "piece-cid", pieceCid, "deal-uuid", dealUuid) + + ctx, span := tracing.Tracer.Start(ctx, "store.remove_deal_for_piece") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.remove-deal-for-piece", "took", time.Since(now).String()) + }(time.Now()) + + s.Lock() + defer s.Unlock() + + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + if errors.Is(err, ds.ErrNotFound) { + return nil + } + return err + } + + for i, v := range md.Deals { + if v.DealUuid == dealUuid { + md.Deals[i] = md.Deals[len(md.Deals)-1] + md.Deals = md.Deals[:len(md.Deals)-1] + break + } + } + + if len(md.Deals) == 0 { + // Remove Metadata if removed deal was last one + if err := s.db.RemovePieceMetadata(ctx, pieceCid); err != nil { + return fmt.Errorf("Failed to remove the Metadata after removing the last deal: %w", err) + } + return nil + } + + err = s.db.SetPieceCidToMetadata(ctx, pieceCid, md) + if err != nil { + return err + } + + return nil +} + +// RemovePieceMetadata remove all Metadata for pieceCID +func (s *Store) RemovePieceMetadata(ctx context.Context, pieceCid cid.Cid) error { + log.Debugw("handle.remove-piece-metadata", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.remove_piece_metadata") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.remove-piece-metadata", "took", time.Since(now).String()) + }(time.Now()) + + s.Lock() + defer s.Unlock() + + if err := s.db.RemovePieceMetadata(ctx, pieceCid); err != nil { + return err + } + + return nil +} + +// RemoveIndexes removes all MultiHashes for pieceCID. To be used manually in case of failure +// in RemoveDealForPiece or RemovePieceMetadata. Metadata for the piece must be +// present in the database +func (s *Store) RemoveIndexes(ctx context.Context, pieceCid cid.Cid) error { + log.Debugw("handle.remove-indexes", "piece-cid", pieceCid) + + ctx, span := tracing.Tracer.Start(ctx, "store.remove_indexes") + defer span.End() + + defer func(now time.Time) { + log.Debugw("handled.remove-indexes", "took", time.Since(now).String()) + }(time.Now()) + + s.Lock() + defer s.Unlock() + + md, err := s.db.GetPieceCidToMetadata(ctx, pieceCid) + if err != nil { + return err + } + + if err := s.db.RemoveIndexes(ctx, md.Cursor, pieceCid); err != nil { + return err + } + + md.IndexedAt = time.Time{} + + err = s.db.SetPieceCidToMetadata(ctx, pieceCid, md) + + return nil +} diff --git a/extern/boostd-data/main.go b/extern/boostd-data/main.go deleted file mode 100644 index 58c4be969..000000000 --- a/extern/boostd-data/main.go +++ /dev/null @@ -1,72 +0,0 @@ -package main - -import ( - "context" - "flag" - "net" - "net/http" - "os" - "os/signal" - "syscall" - "time" - - "github.com/filecoin-project/boost/cmd/boostd-data/svc" - logging "github.com/ipfs/go-log/v2" -) - -var ( - repopath string - db string - - log = logging.Logger("boostd-data") -) - -func init() { - logging.SetLogLevel("*", "debug") - - flag.StringVar(&db, "db", "", "db type for boostd-data (couchbase or ldb)") - flag.StringVar(&repopath, "repopath", "", "path for repo") -} - -func main() { - flag.Parse() - - done := make(chan struct{}) - - srv := svc.New(db, repopath) - addr := "localhost:8089" - ln, err := net.Listen("tcp", addr) - if err != nil { - log.Fatal(err) - } - - log.Infow("server is listening", "addr", "localhost:8089") - - go func() { - err = srv.Serve(ln) - if err != nil && err != http.ErrServerClosed { - log.Fatal(err) - } - - done <- struct{}{} - }() - - // setup a signal handler to cancel the context - interrupt := make(chan os.Signal, 1) - signal.Notify(interrupt, syscall.SIGTERM, syscall.SIGINT) - select { - case <-interrupt: - log.Debugw("got os signal interrupt") - } - - log.Debug("shutting down server") - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - if err := srv.Shutdown(ctx); err != nil { - panic(err) // failure/timeout shutting down the server gracefully - } - - <-done -} diff --git a/extern/boostd-data/model/model.go b/extern/boostd-data/model/model.go index d46cf4b40..a994ef7c2 100644 --- a/extern/boostd-data/model/model.go +++ b/extern/boostd-data/model/model.go @@ -1,10 +1,13 @@ package model import ( + "encoding/base64" + "encoding/binary" + "fmt" "time" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/google/uuid" "github.com/ipfs/go-cid" ) @@ -15,26 +18,73 @@ import ( // Piece ......[ ]...... // CAR ......[ ]............ type DealInfo struct { - DealUuid uuid.UUID `json:"deal_uuid"` - ChainDealID abi.DealID `json:"chain_deal_id"` - SectorID abi.SectorNumber `json:"sector_id"` - PieceOffset abi.PaddedPieceSize `json:"piece_offset"` - PieceLength abi.PaddedPieceSize `json:"piece_length"` + DealUuid string `json:"u"` + IsLegacy bool `json:"y"` + ChainDealID abi.DealID `json:"i"` + MinerAddr address.Address `json:"m"` + SectorID abi.SectorNumber `json:"s"` + PieceOffset abi.PaddedPieceSize `json:"o"` + PieceLength abi.PaddedPieceSize `json:"l"` // The size of the CAR file without zero-padding. // This value may be zero if the size is unknown. - CarLength uint64 `json:"car_length"` + CarLength uint64 `json:"c"` - // If we don't have CarLength, we have to iterate over all offsets, get the largest offset and sum it with length. + // If we don't have CarLength, we have to iterate over all offsets, get + // the largest offset and sum it with length. } // Metadata for PieceCid type Metadata struct { - Cursor uint64 `json:"cursor"` - IndexedAt time.Time `json:"indexed_at"` - Deals []DealInfo `json:"deals"` + Version string `json:"v"` + IndexedAt time.Time `json:"i"` + // CompleteIndex indicates whether the index has all information or is + // missing block size information. Note that indexes imported from the + // dagstore do not have block size information. + CompleteIndex bool `json:"c"` + Deals []DealInfo `json:"d"` + Error string `json:"e"` + ErrorType string `json:"t"` } +// Record is the information stored in the index for each block in a piece type Record struct { - Cid cid.Cid + Cid cid.Cid + OffsetSize +} + +type OffsetSize struct { + // Offset is the offset into the CAR file of the section, where a section + // is
Offset uint64 + // Size is the size of the block data (not the whole section) + Size uint64 +} + +func (ofsz *OffsetSize) MarshallBase64() string { + buf := make([]byte, 2*binary.MaxVarintLen64) + n := binary.PutUvarint(buf, ofsz.Offset) + n += binary.PutUvarint(buf[n:], ofsz.Size) + return base64.RawStdEncoding.EncodeToString(buf[:n]) +} + +func (ofsz *OffsetSize) UnmarshallBase64(str string) error { + buf, err := base64.RawStdEncoding.DecodeString(str) + if err != nil { + return fmt.Errorf("decoding offset/size from base64 string: %w", err) + } + + offset, n := binary.Uvarint(buf) + size, _ := binary.Uvarint(buf[n:]) + + ofsz.Offset = offset + ofsz.Size = size + + return nil +} + +// FlaggedPiece is a piece that has been flagged for the user's attention +// (eg because the index is missing) +type FlaggedPiece struct { + CreatedAt time.Time + PieceCid cid.Cid } diff --git a/extern/boostd-data/model/model_test.go b/extern/boostd-data/model/model_test.go new file mode 100644 index 000000000..2cdee9da2 --- /dev/null +++ b/extern/boostd-data/model/model_test.go @@ -0,0 +1,49 @@ +package model + +import ( + "encoding/json" + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDBEncodeOffsetSize(t *testing.T) { + ofszs := []OffsetSize{{ + Offset: 0, + Size: 0, + }, { + Offset: math.MaxUint64 / 2, + Size: math.MaxUint64 / 2, + }, { + Offset: math.MaxUint64, + Size: math.MaxUint64, + }, { + Offset: 3039040395, + Size: 5345435245, + }} + + for _, ofsz := range ofszs { + jsonbz, err := json.Marshal(ofsz) + require.NoError(t, err) + + t.Logf("%d / %d\n", ofsz.Offset, ofsz.Size) + t.Logf("json encoded: %d bytes\n", len(jsonbz)) + + compressed := ofsz.MarshallBase64() + t.Logf("compressed: %d bytes\n", len(compressed)) + + var res OffsetSize + err = json.Unmarshal(jsonbz, &res) + require.NoError(t, err) + + require.Equal(t, ofsz.Offset, res.Offset) + require.Equal(t, ofsz.Size, res.Size) + + var decompressed OffsetSize + err = decompressed.UnmarshallBase64(compressed) + require.NoError(t, err) + require.Equal(t, ofsz.Offset, decompressed.Offset) + require.Equal(t, ofsz.Size, decompressed.Size) + } +} diff --git a/extern/boostd-data/svc/couchbase_test.go b/extern/boostd-data/svc/couchbase_test.go deleted file mode 100644 index 119951454..000000000 --- a/extern/boostd-data/svc/couchbase_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package svc - -import ( - "io" - "os" - "testing" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - cl "github.com/docker/docker/client" - "github.com/docker/go-connections/nat" - "github.com/filecoin-project/boost/cmd/boostd-data/client" - "github.com/filecoin-project/boost/cmd/boostd-data/model" - "github.com/ipfs/go-cid" - "golang.org/x/net/context" -) - -func XTestCouchbaseService(t *testing.T) { - removeContainer := setupCouchbase(t) - defer removeContainer() - - addr, cleanup := setupService(t, "couchbase") - - cl, err := client.NewStore("http://" + addr) - if err != nil { - t.Fatal(err) - } - - pieceCid, err := cid.Parse("baga6ea4seaqj2j4zfi2xk7okc7fnuw42pip6vjv2tnc4ojsbzlt3rfrdroa7qly") - if err != nil { - t.Fatal(err) - } - dealInfo := model.DealInfo{} - err = cl.AddDealForPiece(pieceCid, dealInfo) - if err != nil { - t.Fatal(err) - } - - log.Debug("sleeping for a while.. running tests..") - time.Sleep(2 * time.Second) - - cleanup() -} - -func setupCouchbase(t *testing.T) func() { - ctx := context.Background() - cli, err := cl.NewEnvClient() - if err != nil { - t.Fatal(err) - } - - imageName := "couchbase" - - out, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{}) - if err != nil { - t.Fatal(err) - } - io.Copy(os.Stdout, out) - - resp, err := cli.ContainerCreate(ctx, &container.Config{ - Image: imageName, - ExposedPorts: nat.PortSet{ - "8091": struct{}{}, - "8092": struct{}{}, - "8093": struct{}{}, - "8094": struct{}{}, - "8095": struct{}{}, - "8096": struct{}{}, - "11210": struct{}{}, - "11211": struct{}{}, - }, - }, &container.HostConfig{ - PortBindings: map[nat.Port][]nat.PortBinding{ - nat.Port("8091"): {{HostIP: "127.0.0.1", HostPort: "8091"}}, - nat.Port("8092"): {{HostIP: "127.0.0.1", HostPort: "8092"}}, - nat.Port("8093"): {{HostIP: "127.0.0.1", HostPort: "8093"}}, - nat.Port("8094"): {{HostIP: "127.0.0.1", HostPort: "8094"}}, - nat.Port("8095"): {{HostIP: "127.0.0.1", HostPort: "8095"}}, - nat.Port("8096"): {{HostIP: "127.0.0.1", HostPort: "8096"}}, - nat.Port("11210"): {{HostIP: "127.0.0.1", HostPort: "11210"}}, - nat.Port("11211"): {{HostIP: "127.0.0.1", HostPort: "11211"}}, - }, - }, nil, nil, "") - - if err != nil { - t.Fatal(err) - } - - if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { - t.Fatal(err) - } - - cleanup := func() { - err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true}) - if err != nil { - t.Fatal(err) - } - } - - // TODO: setup admin password - // TODO: create bucket -- see: https://docs.couchbase.com/server/current/manage/manage-buckets/create-bucket.html - - return cleanup -} diff --git a/extern/boostd-data/svc/setup_couchbase_test_util.go b/extern/boostd-data/svc/setup_couchbase_test_util.go new file mode 100644 index 000000000..5cc989303 --- /dev/null +++ b/extern/boostd-data/svc/setup_couchbase_test_util.go @@ -0,0 +1,255 @@ +package svc + +import ( + "fmt" + "io" + "net/http" + "net/url" + "os" + "testing" + "time" + + "github.com/couchbase/gocb/v2" + "github.com/davecgh/go-spew/spew" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + dockercl "github.com/docker/docker/client" + "github.com/docker/go-connections/nat" + "github.com/filecoin-project/boostd-data/couchbase" + logging "github.com/ipfs/go-log/v2" + "github.com/stretchr/testify/require" + "golang.org/x/net/context" +) + +var tlog = logging.Logger("cbtest") + +func init() { + logging.SetLogLevel("cbtest", "debug") +} + +func SetupCouchbase(t *testing.T, dbSettings couchbase.DBSettings) { + ctx := context.Background() + cli, err := dockercl.NewClientWithOpts(dockercl.FromEnv) + require.NoError(t, err) + + imageName := "couchbase" + out, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{}) + require.NoError(t, err) + + _, err = io.Copy(os.Stdout, out) + require.NoError(t, err) + + tlog.Info("couchbase docker container create...") + resp, err := cli.ContainerCreate(ctx, &container.Config{ + Image: imageName, + ExposedPorts: nat.PortSet{ + "8091": struct{}{}, + "8092": struct{}{}, + "8093": struct{}{}, + "8094": struct{}{}, + "8095": struct{}{}, + "8096": struct{}{}, + "11210": struct{}{}, + "11211": struct{}{}, + }, + }, &container.HostConfig{ + PortBindings: map[nat.Port][]nat.PortBinding{ + nat.Port("8091"): {{HostIP: "127.0.0.1", HostPort: "8091"}}, + nat.Port("8092"): {{HostIP: "127.0.0.1", HostPort: "8092"}}, + nat.Port("8093"): {{HostIP: "127.0.0.1", HostPort: "8093"}}, + nat.Port("8094"): {{HostIP: "127.0.0.1", HostPort: "8094"}}, + nat.Port("8095"): {{HostIP: "127.0.0.1", HostPort: "8095"}}, + nat.Port("8096"): {{HostIP: "127.0.0.1", HostPort: "8096"}}, + nat.Port("11210"): {{HostIP: "127.0.0.1", HostPort: "11210"}}, + nat.Port("11211"): {{HostIP: "127.0.0.1", HostPort: "11211"}}, + }, + }, nil, nil, "") + require.NoError(t, err) + tlog.Info("couchbase docker container created") + + tlog.Info("couchbase docker container start...") + err = cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}) + require.NoError(t, err) + tlog.Info("couchbase docker container started") + + inspect, err := cli.ContainerInspect(ctx, resp.ID) + require.NoError(t, err) + spew.Dump(inspect) + + t.Cleanup(func() { + tlog.Info("couchbase docker container remove...") + err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true}) + require.NoError(t, err) + tlog.Info("couchbase docker container removed") + }) + + tlog.Info("wait for couchbase start...") + awaitCouchbaseUp(t, 10*time.Minute) + tlog.Info("couchbase started") + + tlog.Info("couchbase initialize cluster...") + err = initializeCouchbaseCluster(t, dbSettings) + require.NoError(t, err) + tlog.Info("couchbase initialized cluster") +} + +func initializeCouchbaseCluster(t *testing.T, settings couchbase.DBSettings) error { + couchDir := "/opt/couchbase/var/lib/couchbase" + apiCall(t, "/nodes/self/controller/settings", url.Values{ + "data_path": {couchDir + "/data"}, + "index_path": {couchDir + "/idata"}, + "cbas_path": {couchDir + "/adata"}, + "eventing_path": {couchDir + "e/edata"}, + }) + + apiCall(t, "/node/controller/rename", url.Values{ + "hostname": {"127.0.0.1"}, + }) + + apiCall(t, "/node/controller/setupServices", url.Values{ + "services": {"kv,n1ql,index"}, + }) + + apiCall(t, "/pools/default", url.Values{ + "memoryQuota": {"1024"}, + "indexMemoryQuota": {"512"}, + "ftsMemoryQuota": {"512"}, + }) + + apiCall(t, "/settings/indexes", url.Values{ + "storageMode": {"plasma"}, + }) + + tlog.Info("wait for services start...") + awaitServicesReady(t, settings, time.Minute) + tlog.Info("services started") + + apiCall(t, "/settings/web", url.Values{ + "port": {"8091"}, + "username": {settings.Auth.Username}, + "password": {settings.Auth.Password}, + }) + + tlog.Info("wait for bucket creation and indexing...") + awaitBucketCreationReady(t, settings, 5*time.Minute) + tlog.Info("bucket creation and indexing started") + + return nil +} + +func awaitServicesReady(t *testing.T, settings couchbase.DBSettings, duration time.Duration) { + start := time.Now() + cluster, err := gocb.Connect(settings.ConnectString, gocb.ClusterOptions{ + TimeoutsConfig: gocb.TimeoutsConfig{ + ConnectTimeout: duration, + }, + }) + require.NoError(t, err) + + var res *gocb.DiagnosticsResult + for res == nil || res.State != gocb.ClusterStateOnline { + res, err := cluster.Diagnostics(nil) + require.NoError(t, err) + + allOnline := true + tlog.Info("Services") + for name, svc := range res.Services { + tlog.Info(" " + name + ":") + for _, endpoint := range svc { + tlog.Info(" " + couchbase.ServiceName(endpoint.Type) + ": " + couchbase.EndpointStateName(endpoint.State)) + if endpoint.State != gocb.EndpointStateConnected { + allOnline = false + } + } + } + + if allOnline { + return + } + + if time.Now().Sub(start) > duration { + require.Fail(t, "timed out waiting for couchbase services to come up after "+duration.String()) + } + time.Sleep(time.Second) + } +} + +func awaitBucketCreationReady(t *testing.T, settings couchbase.DBSettings, duration time.Duration) { + start := time.Now() + cluster, err := gocb.Connect(settings.ConnectString, gocb.ClusterOptions{ + TimeoutsConfig: gocb.TimeoutsConfig{ + ConnectTimeout: time.Minute, + }, + Authenticator: gocb.PasswordAuthenticator{ + Username: settings.Auth.Username, + Password: settings.Auth.Password, + }, + }) + require.NoError(t, err) + + // Repeatedly try to create a dummy bucket as a way to ensure that the + // services needed for bucket creation and indexing are up + for { + // It may take several seconds to create the index, so we want to use + // a relatively long timeout here + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := couchbase.CreateBucket(ctx, cluster, "dummy", 128) + if err == nil { + return + } + + if time.Now().Sub(start) > duration { + require.Fail(t, "timed out trying to create dummy bucket after "+duration.String()) + } + + tlog.Infow("got err creating dummy bucket - probably services are not all up yet, retrying", "err", err.Error()) + time.Sleep(time.Second) + } +} + +func awaitCouchbaseUp(t *testing.T, duration time.Duration) { + waitForOkResponse(t, "/internalSettings", duration) +} + +func waitForOkResponse(t *testing.T, path string, duration time.Duration) { + start := time.Now() + for { + fullPath := "http://127.0.0.1:8091" + path + resp, err := http.Get(fullPath) + if resp != nil && resp.Body != nil { + _ = resp.Body.Close() + } + if err == nil && resp != nil { + if resp.StatusCode < 300 { + return + } else if resp.StatusCode >= 400 && resp.StatusCode < 500 { + msg := fmt.Sprintf("failed to GET %s: %d - %s", fullPath, resp.StatusCode, resp.Status) + require.Fail(t, msg) + } + } + + if time.Now().Sub(start) > duration { + msg := "failed to GET " + fullPath + " after " + duration.String() + if resp != nil { + msg += fmt.Sprintf(": %d - %s", resp.StatusCode, resp.Status) + } + if err != nil { + msg += ": " + err.Error() + } + require.Fail(t, msg) + } + time.Sleep(time.Second) + } +} + +func apiCall(t *testing.T, path string, values url.Values) { + fullPath := "http://127.0.0.1:8091" + path + resp, err := http.PostForm(fullPath, values) + require.NoError(t, err) + if resp.StatusCode >= 300 { + require.Fail(t, fmt.Sprintf("%s: %d %s", fullPath, resp.StatusCode, resp.Status)) + } + err = resp.Body.Close() + require.NoError(t, err) +} diff --git a/extern/boostd-data/svc/svc.go b/extern/boostd-data/svc/svc.go index 96bb03c32..6454deb87 100644 --- a/extern/boostd-data/svc/svc.go +++ b/extern/boostd-data/svc/svc.go @@ -5,74 +5,93 @@ import ( "fmt" "net" "net/http" + "os" + "path" "time" - "github.com/ethereum/go-ethereum/rpc" - "github.com/filecoin-project/boost/cmd/boostd-data/couchbase" - "github.com/filecoin-project/boost/cmd/boostd-data/ldb" + "github.com/filecoin-project/boostd-data/couchbase" + "github.com/filecoin-project/boostd-data/ldb" + "github.com/filecoin-project/boostd-data/svc/types" + "github.com/filecoin-project/go-jsonrpc" "github.com/gorilla/mux" logging "github.com/ipfs/go-log/v2" ) var ( - log = logging.Logger("svc") + log = logging.Logger("piecedir") ) -func New(db string, repopath string) *http.Server { - server := rpc.NewServer() - - switch db { - case "couchbase": - ds := couchbase.NewStore() - server.RegisterName("boostddata", ds) - case "ldb": - ds := ldb.NewStore(repopath) - server.RegisterName("boostddata", ds) - default: - panic(fmt.Sprintf("unknown db: %s", db)) - } +type Service struct { + impl types.ServiceImpl +} - router := mux.NewRouter() - router.Handle("/", server) +func NewCouchbase(settings couchbase.DBSettings) *Service { + return &Service{impl: couchbase.NewStore(settings)} +} - log.Infow("server is listening", "addr", "localhost:8089") +func NewLevelDB(repoPath string) (*Service, error) { + if repoPath != "" { // an empty repo path is used for testing + var err error + repoPath, err = MakeLevelDBDir(repoPath) + if err != nil { + return nil, err + } + } - return &http.Server{Handler: router} + return &Service{impl: ldb.NewStore(repoPath)}, nil +} + +func MakeLevelDBDir(repoPath string) (string, error) { + repoPath = path.Join(repoPath, "lid", "leveldb") + if err := os.MkdirAll(repoPath, os.ModePerm); err != nil { + return "", fmt.Errorf("creating leveldb repo directory %s: %w", repoPath, err) + } + return repoPath, nil } -func Setup(db string) (string, func(), error) { - addr := "localhost:0" +func (s *Service) Start(ctx context.Context, port int) error { + addr := fmt.Sprintf("localhost:%d", port) ln, err := net.Listen("tcp", addr) if err != nil { - return "", nil, err + return fmt.Errorf("setting up listener for local index directory service: %w", err) } - srv := New(db, "") - done := make(chan struct{}) + err = s.impl.Start(ctx) + if err != nil { + return fmt.Errorf("starting local index directory service: %w", err) + } + + server := jsonrpc.NewServer() + server.Register("boostddata", s.impl) + router := mux.NewRouter() + router.Handle("/", server) - log.Infow("server is listening", "addr", ln.Addr()) + srv := &http.Server{Handler: router} + log.Infow("local index directory server is listening", "addr", ln.Addr()) + done := make(chan struct{}) go func() { err = srv.Serve(ln) if err != nil && err != http.ErrServerClosed { - panic(err) + log.Errorf("exiting local index directory server: %s", err) } done <- struct{}{} }() - cleanup := func() { + go func() { + <-ctx.Done() log.Debug("shutting down server") - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - if err := srv.Shutdown(ctx); err != nil { - panic(err) + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Errorf("shutting down local index directory server: %s", err) } <-done - } + }() - return ln.Addr().String(), cleanup, nil + return nil } diff --git a/extern/boostd-data/svc/svc_test.go b/extern/boostd-data/svc/svc_test.go index 54de31aeb..ba5795fee 100644 --- a/extern/boostd-data/svc/svc_test.go +++ b/extern/boostd-data/svc/svc_test.go @@ -5,192 +5,310 @@ import ( "bytes" "context" "encoding/hex" - "errors" "fmt" - "net" - "net/http" + "math/rand" "os" "testing" "time" - "github.com/filecoin-project/boost/cmd/boostd-data/client" - "github.com/filecoin-project/boost/cmd/boostd-data/model" + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/boost/testutil" + "github.com/filecoin-project/boostd-data/client" + "github.com/filecoin-project/boostd-data/couchbase" + "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/go-state-types/abi" "github.com/google/uuid" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" + "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/index" "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" ) -func init() { - logging.SetLogLevel("*", "debug") +var testCouchSettings = couchbase.DBSettings{ + ConnectString: "couchbase://localhost", + Auth: couchbase.DBSettingsAuth{ + Username: "Administrator", + Password: "boostdemo", + }, + PieceMetadataBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + MultihashToPiecesBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + PieceOffsetsBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + TestMode: true, } -func TestLdbService(t *testing.T) { - addr, cleanup, err := Setup("ldb") - if err != nil { - t.Fatal(err) - } +func TestService(t *testing.T) { + _ = logging.SetLogLevel("*", "debug") - cl, err := client.NewStore("http://" + addr) - if err != nil { - t.Fatal(err) - } + t.Run("leveldb", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + bdsvc, err := NewLevelDB("") + require.NoError(t, err) + testService(ctx, t, bdsvc, 8042) + }) + t.Run("couchbase", func(t *testing.T) { + // TODO: Unskip this test once the couchbase instance can be created + // from a docker container in CI as part of the test + t.Skip() + // Running couchbase tests may require download the docker container + // so set a high timeout + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + SetupCouchbase(t, testCouchSettings) + bdsvc := NewCouchbase(testCouchSettings) + testService(ctx, t, bdsvc, 8043) + }) +} + +func testService(ctx context.Context, t *testing.T, bdsvc *Service, port int) { + err := bdsvc.Start(ctx, port) + require.NoError(t, err) + + cl := client.NewStore() + err = cl.Dial(context.Background(), fmt.Sprintf("http://localhost:%d", port)) + require.NoError(t, err) + defer cl.Close(ctx) sampleidx := "fixtures/baga6ea4seaqnfhocd544oidrgsss2ahoaomvxuaqxfmlsizljtzsuivjl5hamka.full.idx" pieceCid, err := cid.Parse("baga6ea4seaqnfhocd544oidrgsss2ahoaomvxuaqxfmlsizljtzsuivjl5hamka") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) subject, err := loadIndex(sampleidx) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) records, err := getRecords(subject) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) - randomuuid := uuid.New() + randomuuid, err := uuid.Parse("4d8f5ce6-dbfd-40dc-8b03-29308e97357b") + require.NoError(t, err) - err = cl.AddIndex(pieceCid, records) - if err != nil { - t.Fatal(err) - } + err = cl.AddIndex(ctx, pieceCid, records, true) + require.NoError(t, err) di := model.DealInfo{ - DealUuid: randomuuid, + DealUuid: randomuuid.String(), SectorID: abi.SectorNumber(1), PieceOffset: 1, PieceLength: 2, CarLength: 3, } - err = cl.AddDealForPiece(pieceCid, di) - if err != nil { - t.Fatal(err) - } + // Add a deal for the piece + err = cl.AddDealForPiece(ctx, pieceCid, di) + require.NoError(t, err) + + // Add the same deal a second time to test uniqueness + err = cl.AddDealForPiece(ctx, pieceCid, di) + require.NoError(t, err) + + // There should only be one deal + dis, err := cl.GetPieceDeals(ctx, pieceCid) + require.NoError(t, err) + require.Len(t, dis, 1) + require.Equal(t, di, dis[0]) b, err := hex.DecodeString("1220ff63d7689e2d9567d1a90a7a68425f430137142e1fbc28fe4780b9ee8a5ef842") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) mhash, err := multihash.Cast(b) - if err != nil { - t.Fatal(err) - } - - offset, err := cl.GetOffset(pieceCid, mhash) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) - if offset != 3039040395 { - t.Fatal("got wrong offset") - } + offset, err := cl.GetOffsetSize(ctx, pieceCid, mhash) + require.NoError(t, err) + require.EqualValues(t, 3039040395, offset.Offset) + require.EqualValues(t, 0, offset.Size) - pcids, err := cl.PiecesContaining(mhash) + pcids, err := cl.PiecesContainingMultihash(ctx, mhash) + require.NoError(t, err) + require.Len(t, pcids, 1) + require.Equal(t, pieceCid, pcids[0]) - if len(pcids) != 1 { - t.Fatalf("expected len of 1 for pieceCids, got: %d", len(pcids)) - } + allPieceCids, err := cl.ListPieces(ctx) + require.NoError(t, err) + require.Len(t, allPieceCids, 1) + require.Equal(t, pieceCid, allPieceCids[0]) - if !pcids[0].Equals(pieceCid) { - t.Fatal("expected for pieceCids to match") - } + indexed, err := cl.IsIndexed(ctx, pieceCid) + require.NoError(t, err) + require.True(t, indexed) - dis, err := cl.GetPieceDeals(pieceCid) + recs, err := cl.GetRecords(ctx, pieceCid) + require.NoError(t, err) + require.Equal(t, len(records), len(recs)) - if len(dis) != 1 { - t.Fatalf("expected len of 1 for dis, got: %d", len(dis)) - } + loadedSubject, err := cl.GetIndex(ctx, pieceCid) + require.NoError(t, err) - if dis[0] != di { - t.Fatal("expected for dealInfos to match") - } + ok, err := compareIndices(subject, loadedSubject) + require.NoError(t, err) + require.True(t, ok) +} - indexed, err := cl.IsIndexed(pieceCid) - if err != nil { - t.Fatal(err) - } +func TestServiceFuzz(t *testing.T) { + t.Skip() + _ = logging.SetLogLevel("*", "info") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + t.Run("level db", func(t *testing.T) { + bdsvc, err := NewLevelDB("") + require.NoError(t, err) + testServiceFuzz(ctx, t, bdsvc, 8042) + }) + t.Run("couchbase", func(t *testing.T) { + // TODO: Unskip this test once the couchbase instance can be created + // from a docker container in CI as part of the test + t.Skip() + SetupCouchbase(t, testCouchSettings) + bdsvc := NewCouchbase(testCouchSettings) + testServiceFuzz(ctx, t, bdsvc, 8043) + }) +} - if !indexed { - t.Fatal("expected pieceCid to be indexed") - } +func testServiceFuzz(ctx context.Context, t *testing.T, bdsvc *Service, port int) { + err := bdsvc.Start(ctx, port) + require.NoError(t, err) - recs, err := cl.GetRecords(pieceCid) - if err != nil { - t.Fatal(err) - } + cl := client.NewStore() + err = cl.Dial(context.Background(), "http://localhost:8042") + require.NoError(t, err) + defer cl.Close(ctx) - if len(recs) == 0 { - t.Fatal("expected to get records back from GetIndex") + var idxs []index.Index + for i := 0; i < 10; i++ { + size := (5 + (i % 3)) << 20 + idxs = append(idxs, createCarIndex(t, size, i+1)) } - loadedSubject, err := cl.GetIndex(pieceCid) - if err != nil { - t.Fatal(err) - } + throttle := make(chan struct{}, 64) + var eg errgroup.Group + for _, idx := range idxs { + idx := idx + eg.Go(func() error { + records, err := getRecords(idx) + require.NoError(t, err) + + randomuuid := uuid.New() + pieceCid := testutil.GenerateCid() + err = cl.AddIndex(ctx, pieceCid, records, true) + require.NoError(t, err) + + di := model.DealInfo{ + DealUuid: randomuuid.String(), + SectorID: abi.SectorNumber(1), + PieceOffset: 1, + PieceLength: 2, + CarLength: 3, + } + + err = cl.AddDealForPiece(ctx, pieceCid, di) + require.NoError(t, err) + + dis, err := cl.GetPieceDeals(ctx, pieceCid) + require.NoError(t, err) + require.Len(t, dis, 1) + require.Equal(t, di, dis[0]) + + indexed, err := cl.IsIndexed(ctx, pieceCid) + require.NoError(t, err) + require.True(t, indexed) + + recs, err := cl.GetRecords(ctx, pieceCid) + require.NoError(t, err) + require.Equal(t, len(records), len(recs)) + + var offsetEG errgroup.Group + for _, r := range recs { + if rand.Float32() > 0.1 { + continue + } + + idx := idx + c := r.Cid + throttle <- struct{}{} + offsetEG.Go(func() error { + defer func() { <-throttle }() + + mhash := c.Hash() + var err error + err1 := idx.GetAll(c, func(expected uint64) bool { + var offsetSize *model.OffsetSize + offsetSize, err = cl.GetOffsetSize(ctx, pieceCid, mhash) + if err != nil { + return false + } + if expected != offsetSize.Offset { + err = fmt.Errorf("cid %s: expected offset %d, got offset %d", c, expected, offsetSize.Offset) + return false + } + return true + }) + if err != nil { + return err + } + if err1 != nil { + return err1 + } + + pcids, err := cl.PiecesContainingMultihash(ctx, mhash) + if err != nil { + return err + } + if len(pcids) != 1 { + return fmt.Errorf("expected 1 piece, got %d", len(pcids)) + } + if pieceCid != pcids[0] { + return fmt.Errorf("expected piece %s, got %s", pieceCid, pcids[0]) + } + return nil + }) + } + err = offsetEG.Wait() + require.NoError(t, err) + + loadedSubject, err := cl.GetIndex(ctx, pieceCid) + require.NoError(t, err) + + ok, err := compareIndices(idx, loadedSubject) + require.NoError(t, err) + require.True(t, ok) - ok, err := compareIndices(subject, loadedSubject) - if err != nil { - t.Fatal(err) - } - if !ok { - log.Fatal("compare failed") + return nil + }) } - log.Debug("sleeping for a while.. running tests..") - - cleanup() + err = eg.Wait() + require.NoError(t, err) } -func setupService(t *testing.T, db string) (string, func()) { - addr := "localhost:0" - ln, err := net.Listen("tcp", addr) - if err != nil { - t.Fatal(err) - } - srv := New(db, "") - - done := make(chan struct{}) - - log.Infow("server is listening", "addr", ln.Addr()) - - go func() { - err = srv.Serve(ln) - if err != nil && err != http.ErrServerClosed { - log.Fatal(err) - } - - done <- struct{}{} - }() - - cleanup := func() { - log.Debug("shutting down server") - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - if err := srv.Shutdown(ctx); err != nil { - panic(err) // failure/timeout shutting down the server gracefully - } - - <-done - } - - return ln.Addr().String(), cleanup +func createCarIndex(t *testing.T, size int, rseed int) index.Index { + // Create a CAR file + randomFilePath, err := testutil.CreateRandomFile(t.TempDir(), rseed, size) + require.NoError(t, err) + _, carFilePath, err := testutil.CreateDenseCARv2(t.TempDir(), randomFilePath) + require.NoError(t, err) + carFile, err := os.Open(carFilePath) + require.NoError(t, err) + defer carFile.Close() + idx, err := car.ReadOrGenerateIndex(carFile) + require.NoError(t, err) + return idx } func loadIndex(path string) (index.Index, error) { defer func(now time.Time) { - log.Debugw("loadindex", "took", fmt.Sprintf("%s", time.Since(now))) + log.Debugw("loadindex", "took", time.Since(now).String()) }(time.Now()) idxf, err := os.Open(path) @@ -217,8 +335,11 @@ func getRecords(subject index.Index) ([]model.Record, error) { cid := cid.NewCidV1(cid.Raw, m) records = append(records, model.Record{ - Cid: cid, - Offset: offset, + Cid: cid, + OffsetSize: model.OffsetSize{ + Offset: offset, + Size: 0, + }, }) return nil @@ -227,7 +348,7 @@ func getRecords(subject index.Index) ([]model.Record, error) { return nil, err } default: - return nil, errors.New(fmt.Sprintf("wanted %v but got %v\n", multicodec.CarMultihashIndexSorted, idx.Codec())) + return nil, fmt.Errorf("wanted %v but got %v\n", multicodec.CarMultihashIndexSorted, idx.Codec()) } return records, nil } @@ -236,14 +357,136 @@ func compareIndices(subject, subjectDb index.Index) (bool, error) { var b bytes.Buffer w := bufio.NewWriter(&b) - subject.Marshal(w) + _, err := subject.Marshal(w) + if err != nil { + return false, err + } var b2 bytes.Buffer w2 := bufio.NewWriter(&b2) - subjectDb.Marshal(w2) + _, err = subjectDb.Marshal(w2) + if err != nil { + return false, err + } res := bytes.Compare(b.Bytes(), b2.Bytes()) return res == 0, nil } + +func TestCleanup(t *testing.T) { + _ = logging.SetLogLevel("*", "debug") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + t.Run("level db", func(t *testing.T) { + bdsvc, err := NewLevelDB("") + require.NoError(t, err) + testCleanup(ctx, t, bdsvc, 8042) + }) + t.Run("couchbase", func(t *testing.T) { + // TODO: Unskip this test once the couchbase instance can be created + // from a docker container in CI as part of the test + t.Skip() + SetupCouchbase(t, testCouchSettings) + bdsvc := NewCouchbase(testCouchSettings) + testCleanup(ctx, t, bdsvc, 8043) + }) +} + +func testCleanup(ctx context.Context, t *testing.T, bdsvc *Service, port int) { + err := bdsvc.Start(ctx, port) + require.NoError(t, err) + + cl := client.NewStore() + err = cl.Dial(context.Background(), fmt.Sprintf("http://localhost:%d", port)) + require.NoError(t, err) + defer cl.Close(ctx) + + sampleidx := "fixtures/baga6ea4seaqnfhocd544oidrgsss2ahoaomvxuaqxfmlsizljtzsuivjl5hamka.full.idx" + + pieceCid, err := cid.Parse("baga6ea4seaqnfhocd544oidrgsss2ahoaomvxuaqxfmlsizljtzsuivjl5hamka") + require.NoError(t, err) + + subject, err := loadIndex(sampleidx) + require.NoError(t, err) + + records, err := getRecords(subject) + require.NoError(t, err) + + randomuuid, err := uuid.Parse("4d8f5ce6-dbfd-40dc-8b03-29308e97357b") + require.NoError(t, err) + + err = cl.AddIndex(ctx, pieceCid, records, true) + require.NoError(t, err) + + di := model.DealInfo{ + DealUuid: randomuuid.String(), + SectorID: abi.SectorNumber(1), + PieceOffset: 1, + PieceLength: 2, + CarLength: 3, + } + + // Add a deal for the piece + err = cl.AddDealForPiece(ctx, pieceCid, di) + require.NoError(t, err) + + // There should only be one deal + dis, err := cl.GetPieceDeals(ctx, pieceCid) + require.NoError(t, err) + require.Len(t, dis, 1) + require.Equal(t, di, dis[0]) + + b, err := hex.DecodeString("1220ff63d7689e2d9567d1a90a7a68425f430137142e1fbc28fe4780b9ee8a5ef842") + require.NoError(t, err) + + mhash, err := multihash.Cast(b) + require.NoError(t, err) + + offset, err := cl.GetOffsetSize(ctx, pieceCid, mhash) + require.NoError(t, err) + require.EqualValues(t, 3039040395, offset.Offset) + require.EqualValues(t, 0, offset.Size) + + pcids, err := cl.PiecesContainingMultihash(ctx, mhash) + require.NoError(t, err) + require.Len(t, pcids, 1) + require.Equal(t, pieceCid, pcids[0]) + + indexed, err := cl.IsIndexed(ctx, pieceCid) + require.NoError(t, err) + require.True(t, indexed) + + recs, err := cl.GetRecords(ctx, pieceCid) + require.NoError(t, err) + require.Equal(t, len(records), len(recs)) + + loadedSubject, err := cl.GetIndex(ctx, pieceCid) + require.NoError(t, err) + + ok, err := compareIndices(subject, loadedSubject) + require.NoError(t, err) + require.True(t, ok) + + err = cl.RemoveDealForPiece(ctx, pieceCid, di.DealUuid) + require.NoError(t, err) + + _, err = cl.GetPieceDeals(ctx, pieceCid) + require.ErrorContains(t, err, "not found") + + _, err = cl.GetOffsetSize(ctx, pieceCid, mhash) + require.ErrorContains(t, err, "not found") + + _, err = cl.GetRecords(ctx, pieceCid) + require.ErrorContains(t, err, "not found") + + _, err = cl.PiecesContainingMultihash(ctx, mhash) + require.ErrorContains(t, err, "not found") + + indexed, err = cl.IsIndexed(ctx, pieceCid) + require.NoError(t, err) + require.False(t, indexed) +} diff --git a/extern/boostd-data/svc/types/types.go b/extern/boostd-data/svc/types/types.go new file mode 100644 index 000000000..631cf27d7 --- /dev/null +++ b/extern/boostd-data/svc/types/types.go @@ -0,0 +1,50 @@ +package types + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/filecoin-project/boostd-data/model" + "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" +) + +var ErrNotFound = errors.New("not found") + +// IsNotFound just does a string match against the error message +// to check if it matches the ErrNotFound error message. +// We have to do string matching so that it can be used on errors that +// cross the RPC boundary (we can't use errors.Is) +func IsNotFound(err error) bool { + return strings.Contains(err.Error(), ErrNotFound.Error()) +} + +type Service interface { + AddDealForPiece(context.Context, cid.Cid, model.DealInfo) error + AddIndex(context.Context, cid.Cid, []model.Record, bool) error + GetIndex(context.Context, cid.Cid) ([]model.Record, error) + IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) + IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, error) + GetOffsetSize(context.Context, cid.Cid, mh.Multihash) (*model.OffsetSize, error) + ListPieces(ctx context.Context) ([]cid.Cid, error) + GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) + GetPieceDeals(context.Context, cid.Cid) ([]model.DealInfo, error) + SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error + IndexedAt(context.Context, cid.Cid) (time.Time, error) + PiecesContainingMultihash(context.Context, mh.Multihash) ([]cid.Cid, error) + RemoveDealForPiece(context.Context, cid.Cid, string) error + RemovePieceMetadata(context.Context, cid.Cid) error + RemoveIndexes(context.Context, cid.Cid) error + NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) + FlagPiece(ctx context.Context, pieceCid cid.Cid) error + UnflagPiece(ctx context.Context, pieceCid cid.Cid) error + FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) + FlaggedPiecesCount(ctx context.Context) (int, error) +} + +type ServiceImpl interface { + Service + Start(ctx context.Context) error +} diff --git a/go.mod b/go.mod index d68a12946..76d3519a5 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( github.com/ipfs/go-metrics-interface v0.0.1 github.com/ipfs/go-unixfs v0.4.0 github.com/ipld/go-car v0.5.0 - github.com/ipld/go-car/v2 v2.5.0 + github.com/ipld/go-car/v2 v2.5.1 github.com/ipld/go-ipld-prime v0.20.0 github.com/ipld/go-ipld-selector-text-lite v0.0.1 github.com/ipni/index-provider v0.10.1 @@ -87,7 +87,9 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multiaddr v0.8.0 github.com/multiformats/go-multibase v0.1.1 + github.com/multiformats/go-multicodec v0.8.0 github.com/multiformats/go-multihash v0.2.1 + github.com/multiformats/go-varint v0.0.6 // indirect github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/pressly/goose/v3 v3.5.3 github.com/prometheus/client_golang v1.13.0 @@ -111,6 +113,7 @@ require ( golang.org/x/text v0.3.7 golang.org/x/tools v0.1.12 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 + google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 ) @@ -160,11 +163,11 @@ require ( github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 // indirect github.com/filecoin-project/go-commp-utils/nonffi v0.0.0-20220905160352-62059082a837 // indirect github.com/filecoin-project/go-crypto v0.0.1 // indirect - github.com/filecoin-project/go-ds-versioning v0.1.2 // indirect + github.com/filecoin-project/go-ds-versioning v0.1.2 github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 // indirect github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 // indirect - github.com/filecoin-project/go-statemachine v1.0.2 // indirect + github.com/filecoin-project/go-statemachine v1.0.2 github.com/filecoin-project/go-storedcounter v0.1.0 // indirect github.com/filecoin-project/pubsub v1.0.0 // indirect github.com/filecoin-project/specs-actors/v2 v2.3.6 @@ -185,10 +188,10 @@ require ( github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.5 // indirect - github.com/go-openapi/jsonpointer v0.19.3 // indirect - github.com/go-openapi/jsonreference v0.19.4 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect github.com/go-openapi/spec v0.19.11 // indirect - github.com/go-openapi/swag v0.19.11 // indirect + github.com/go-openapi/swag v0.19.14 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -289,9 +292,7 @@ require ( github.com/multiformats/go-base36 v0.1.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect - github.com/multiformats/go-multicodec v0.8.0 // indirect github.com/multiformats/go-multistream v0.3.3 // indirect - github.com/multiformats/go-varint v0.0.6 // indirect github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c // indirect github.com/nkovacs/streamquote v1.0.0 // indirect github.com/nxadm/tail v1.4.8 // indirect @@ -320,7 +321,7 @@ require ( github.com/twmb/murmur3 v1.1.6 // indirect github.com/ugorji/go/codec v1.2.6 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasttemplate v1.0.1 // indirect + github.com/valyala/fasttemplate v1.2.1 // indirect github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect @@ -332,14 +333,13 @@ require ( github.com/zondax/hid v0.9.1 // indirect github.com/zondax/ledger-go v0.12.1 // indirect go.uber.org/dig v1.12.0 // indirect - go.uber.org/zap v1.23.0 // indirect + go.uber.org/zap v1.23.0 go4.org v0.0.0-20200411211856-f5505b9728dd // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5 // indirect golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect - google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect @@ -352,11 +352,20 @@ require ( require ( github.com/filecoin-project/boostd-data v0.0.0-00010101000000-000000000000 github.com/ipfs/go-ds-flatfs v0.5.1 + github.com/schollz/progressbar/v3 v3.11.0 ) require ( + github.com/Microsoft/go-winio v0.5.2 // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - go.etcd.io/bbolt v1.3.6 // indirect + github.com/containerd/containerd v1.6.10 // indirect + github.com/couchbase/gocb/v2 v2.5.0 // indirect + github.com/couchbase/gocbcore/v10 v10.1.2 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/docker v20.10.7+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect ) diff --git a/go.sum b/go.sum index 2d6711cdc..f5966b8a9 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbL github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= @@ -75,6 +76,8 @@ github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2o github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= @@ -214,6 +217,8 @@ github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJ github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/containerd v1.6.10 h1:8aiav7I2ZyQLbTlNMcBXyAU1FtFvp6VuyuW13qSd6Hk= +github.com/containerd/containerd v1.6.10/go.mod h1:CVqfxdJ95PDgORwA219AwwLrREZgrTFybXu2HfMKRG0= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -232,6 +237,12 @@ github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= +github.com/couchbase/gocb/v2 v2.5.0 h1:BcNfLLeSs2pCppCK4wOW9aqDamQTSUMiwUZQCZvLnQo= +github.com/couchbase/gocb/v2 v2.5.0/go.mod h1:xIF7+2G9uANWSJVv151NwM8lr+5cluxaOx20Fvlt8nw= +github.com/couchbase/gocbcore/v10 v10.1.2 h1:u4OmKmub4soyf/cXMXaQohTXruJAxBL9wN6ke3DLe78= +github.com/couchbase/gocbcore/v10 v10.1.2/go.mod h1:qkPnOBziCs0guMEEvd0cRFo+AjOW0yEL99cU3I4n3Ao= +github.com/couchbaselabs/gocaves/client v0.0.0-20220223122017-22859b310bd2 h1:UlwJ2GWpZQAQCLHyO3xHKcqAjUUcX2w7FKpbxCIUQks= +github.com/couchbaselabs/gocaves/client v0.0.0-20220223122017-22859b310bd2/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -282,7 +293,11 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -480,20 +495,23 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.4 h1:3Vw+rh13uq2JFNxgnMTGE1rnoieU9FmyE1gvnyylsYg= github.com/go-openapi/jsonreference v0.19.4/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/spec v0.19.7/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/spec v0.19.11 h1:ogU5q8dtp3MMPn59a9VRrPKVxvJHEs5P7yNMR5sNnis= github.com/go-openapi/spec v0.19.11/go.mod h1:vqK/dIdLGCosfvYsQV3WfC7N3TiZSnGY2RZKoFK7X28= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.8/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= -github.com/go-openapi/swag v0.19.11 h1:RFTu/dlFySpyVvJDfp/7674JY4SDglYWKztbiIGFpmc= github.com/go-openapi/swag v0.19.11/go.mod h1:Uc0gKkdR+ojzsEpjh39QChyu92vPgIr72POcgHMAgSY= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -635,7 +653,6 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= @@ -921,8 +938,8 @@ github.com/ipld/go-car v0.5.0 h1:kcCEa3CvYMs0iE5BzD5sV7O2EwMiCIp3uF8tA6APQT8= github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= github.com/ipld/go-car/v2 v2.4.1/go.mod h1:zjpRf0Jew9gHqSvjsKVyoq9OY9SWoEKdYCQUKVaaPT0= -github.com/ipld/go-car/v2 v2.5.0 h1:S9h7A6qBAJ+B1M1jIKtau+HPDe30UbM71vsyBzwvRIE= -github.com/ipld/go-car/v2 v2.5.0/go.mod h1:jKjGOqoCj5zn6KjnabD6JbnCsMntqU2hLiU6baZVO3E= +github.com/ipld/go-car/v2 v2.5.1 h1:U2ux9JS23upEgrJScW8VQuxmE94560kYxj9CQUpcfmk= +github.com/ipld/go-car/v2 v2.5.1/go.mod h1:jKjGOqoCj5zn6KjnabD6JbnCsMntqU2hLiU6baZVO3E= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY= @@ -1002,6 +1019,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3/go.mod h1:BYpt4ufZiIGv2nXn4gMxnfKV306n3mWXgNu/d2TqdTU= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -1479,6 +1497,8 @@ github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -1490,12 +1510,14 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -1590,6 +1612,7 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c h1:5bFTChQxSKNwy8ALwOebjekYExl9HTT9urdawqC95tA= github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c/go.mod h1:7qN3Y0BvzRUf4LofcoJplQL10lsFDb4PYlePTVwrP28= github.com/nkovacs/streamquote v1.0.0 h1:PmVIV08Zlx2lZK5fFZlMZ04eHcDTIFJCv/5/0twVUow= @@ -1623,8 +1646,11 @@ github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWEr github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 h1:CznVS40zms0Dj5he4ERo+fRPtO0qxUk8lA8Xu3ddet0= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= +github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= @@ -1765,6 +1791,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/schollz/progressbar/v3 v3.11.0 h1:3nIBUF1Zw/pGUaRHP7PZWmARP7ZQbWQ6vL6hwoQiIvU= +github.com/schollz/progressbar/v3 v3.11.0/go.mod h1:R2djRgv58sn00AGysc4fN0ip4piOGd3z88K+zVBjczs= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sercand/kuberesolver v2.1.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ= @@ -1841,6 +1869,7 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -1899,8 +1928,9 @@ github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU= github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= @@ -1989,7 +2019,6 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -2332,7 +2361,6 @@ golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2376,6 +2404,7 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc= golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2530,7 +2559,6 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -2592,6 +2620,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= @@ -2618,6 +2647,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -2625,6 +2655,7 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/gql/resolver.go b/gql/resolver.go index 4cdd422c0..994bcfa91 100644 --- a/gql/resolver.go +++ b/gql/resolver.go @@ -13,6 +13,7 @@ import ( gqltypes "github.com/filecoin-project/boost/gql/types" "github.com/filecoin-project/boost/markets/storageadapter" "github.com/filecoin-project/boost/node/config" + "github.com/filecoin-project/boost/piecedirectory" "github.com/filecoin-project/boost/retrievalmarket/rtvllog" "github.com/filecoin-project/boost/storagemanager" "github.com/filecoin-project/boost/storagemarket" @@ -20,7 +21,6 @@ import ( "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints" "github.com/filecoin-project/boost/transport" - "github.com/filecoin-project/dagstore" "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" lotus_storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" @@ -43,48 +43,52 @@ type dealListResolver struct { // resolver translates from a request for a graphql field to the data for // that field type resolver struct { - cfg *config.Boost - repo lotus_repo.LockedRepo - h host.Host - dealsDB *db.DealsDB - logsDB *db.LogsDB - retDB *rtvllog.RetrievalLogDB - plDB *db.ProposalLogsDB - fundsDB *db.FundsDB - fundMgr *fundmanager.FundManager - storageMgr *storagemanager.StorageManager - provider *storagemarket.Provider - legacyProv lotus_storagemarket.StorageProvider - legacyDT lotus_dtypes.ProviderDataTransfer - ps piecestore.PieceStore - sa retrievalmarket.SectorAccessor - dagst dagstore.Interface - publisher *storageadapter.DealPublisher - spApi sealingpipeline.API - fullNode v1api.FullNode -} - -func NewResolver(cfg *config.Boost, r lotus_repo.LockedRepo, h host.Host, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, spApi sealingpipeline.API, provider *storagemarket.Provider, legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, ps piecestore.PieceStore, sa retrievalmarket.SectorAccessor, dagst dagstore.Interface, publisher *storageadapter.DealPublisher, fullNode v1api.FullNode) *resolver { + // This context is closed when boost shuts down + ctx context.Context + + cfg *config.Boost + repo lotus_repo.LockedRepo + h host.Host + dealsDB *db.DealsDB + logsDB *db.LogsDB + retDB *rtvllog.RetrievalLogDB + plDB *db.ProposalLogsDB + fundsDB *db.FundsDB + fundMgr *fundmanager.FundManager + storageMgr *storagemanager.StorageManager + provider *storagemarket.Provider + legacyProv lotus_storagemarket.StorageProvider + legacyDT lotus_dtypes.ProviderDataTransfer + ps piecestore.PieceStore + sa retrievalmarket.SectorAccessor + piecedirectory *piecedirectory.PieceDirectory + publisher *storageadapter.DealPublisher + spApi sealingpipeline.API + fullNode v1api.FullNode +} + +func NewResolver(ctx context.Context, cfg *config.Boost, r lotus_repo.LockedRepo, h host.Host, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, spApi sealingpipeline.API, provider *storagemarket.Provider, legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, ps piecestore.PieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, publisher *storageadapter.DealPublisher, fullNode v1api.FullNode) *resolver { return &resolver{ - cfg: cfg, - repo: r, - h: h, - dealsDB: dealsDB, - logsDB: logsDB, - retDB: retDB, - plDB: plDB, - fundsDB: fundsDB, - fundMgr: fundMgr, - storageMgr: storageMgr, - provider: provider, - legacyProv: legacyProv, - legacyDT: legacyDT, - ps: ps, - sa: sa, - dagst: dagst, - publisher: publisher, - spApi: spApi, - fullNode: fullNode, + ctx: ctx, + cfg: cfg, + repo: r, + h: h, + dealsDB: dealsDB, + logsDB: logsDB, + retDB: retDB, + plDB: plDB, + fundsDB: fundsDB, + fundMgr: fundMgr, + storageMgr: storageMgr, + provider: provider, + legacyProv: legacyProv, + legacyDT: legacyDT, + ps: ps, + sa: sa, + piecedirectory: piecedirectory, + publisher: publisher, + spApi: spApi, + fullNode: fullNode, } } diff --git a/gql/resolver_funds.go b/gql/resolver_funds.go index 360c1e6c3..6484881e5 100644 --- a/gql/resolver_funds.go +++ b/gql/resolver_funds.go @@ -3,8 +3,6 @@ package gql import ( "context" "fmt" - "time" - gqltypes "github.com/filecoin-project/boost/gql/types" smfunds "github.com/filecoin-project/boost/storagemarket/funds" "github.com/graph-gophers/graphql-go" @@ -84,15 +82,9 @@ func (r *resolver) FundsLogs(ctx context.Context, args fundsLogsArgs) (*fundsLog limit = int(*args.Limit.Value) } - var cursor *time.Time - if args.Cursor != nil { - val := (*args.Cursor).Int64() - asTime := time.Unix(val/1000, (val%1000)*1e6) - cursor = &asTime - } - // Fetch one extra log so that we can check if there are more logs // beyond the limit + cursor := bigIntToTime(args.Cursor) logs, err := r.fundsDB.Logs(ctx, cursor, offset, limit+1) if err != nil { return nil, fmt.Errorf("getting funds logs: %w", err) diff --git a/gql/resolver_piece.go b/gql/resolver_piece.go index 8c9e96716..52d952ff7 100644 --- a/gql/resolver_piece.go +++ b/gql/resolver_piece.go @@ -2,47 +2,54 @@ package gql import ( "context" - "errors" "fmt" - + "time" + gqltypes "github.com/filecoin-project/boost/gql/types" - "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints" - "github.com/filecoin-project/dagstore" - "github.com/filecoin-project/dagstore/shard" + pdtypes "github.com/filecoin-project/boost/piecedirectory/types" + "github.com/filecoin-project/boostd-data/svc/types" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin/v9/market" "github.com/graph-gophers/graphql-go" "github.com/ipfs/go-cid" - "github.com/ipfs/go-datastore" + "github.com/multiformats/go-multihash" ) type IndexStatus string const ( - IndexStatusUnknown IndexStatus = "" IndexStatusNotFound IndexStatus = "NotFound" IndexStatusRegistered IndexStatus = "Registered" + IndexStatusIndexing IndexStatus = "Indexing" IndexStatusComplete IndexStatus = "Complete" IndexStatusFailed IndexStatus = "Failed" ) +type sealStatusResolver struct { + IsUnsealed bool + Error string +} + type pieceDealResolver struct { - Deal *basicDealResolver - Sector *sectorResolver - SealStatus *sealStatus + sa retrievalmarket.SectorAccessor + Deal *basicDealResolver + Sector *sectorResolver } -type sealStatus struct { - IsUnsealed bool - Error string +func (pdr *pieceDealResolver) SealStatus(ctx context.Context) *sealStatusResolver { + return sealStatus(ctx, pdr.sa, pdr.Sector) } type pieceInfoDeal struct { + sa retrievalmarket.SectorAccessor ChainDealID gqltypes.Uint64 Sector *sectorResolver - SealStatus *sealStatus +} + +func (pid *pieceInfoDeal) SealStatus(ctx context.Context) *sealStatusResolver { + return sealStatus(ctx, pid.sa, pid.Sector) } type indexStatus struct { @@ -57,15 +64,86 @@ type pieceResolver struct { PieceInfoDeals []*pieceInfoDeal } +type flaggedPieceResolver struct { + Piece *pieceResolver + CreatedAt graphql.Time +} + +type piecesFlaggedArgs struct { + Cursor *gqltypes.BigInt // CreatedAt in milli-seconds + Offset graphql.NullInt + Limit graphql.NullInt +} + +type flaggedPieceListResolver struct { + TotalCount int32 + Pieces []*flaggedPieceResolver + More bool +} + +func (r *resolver) PiecesFlagged(ctx context.Context, args piecesFlaggedArgs) (*flaggedPieceListResolver, error) { + offset := 0 + if args.Offset.Set && args.Offset.Value != nil && *args.Offset.Value > 0 { + offset = int(*args.Offset.Value) + } + + limit := 10 + if args.Limit.Set && args.Limit.Value != nil && *args.Limit.Value > 0 { + limit = int(*args.Limit.Value) + } + + // Fetch one extra row so that we can check if there are more rows + // beyond the limit + cursor := bigIntToTime(args.Cursor) + flaggedPieces, err := r.piecedirectory.FlaggedPiecesList(ctx, cursor, offset, limit+1) + if err != nil { + return nil, err + } + more := len(flaggedPieces) > limit + if more { + // Truncate list to limit + flaggedPieces = flaggedPieces[:limit] + } + + // Get the total row count + count, err := r.piecedirectory.FlaggedPiecesCount(ctx) + if err != nil { + return nil, err + } + + allLegacyDeals, err := r.legacyProv.ListLocalDeals() + if err != nil { + return nil, err + } + + flaggedPieceResolvers := make([]*flaggedPieceResolver, 0, len(flaggedPieces)) + for _, flaggedPiece := range flaggedPieces { + pieceResolver, err := r.pieceStatus(ctx, flaggedPiece.PieceCid, allLegacyDeals) + if err != nil { + return nil, err + } + flaggedPieceResolvers = append(flaggedPieceResolvers, &flaggedPieceResolver{ + Piece: pieceResolver, + CreatedAt: graphql.Time{Time: flaggedPiece.CreatedAt}, + }) + } + + return &flaggedPieceListResolver{ + TotalCount: int32(count), + Pieces: flaggedPieceResolvers, + More: more, + }, nil +} + func (r *resolver) PiecesWithPayloadCid(ctx context.Context, args struct{ PayloadCid string }) ([]string, error) { payloadCid, err := cid.Parse(args.PayloadCid) if err != nil { return nil, fmt.Errorf("%s is not a valid payload cid", args.PayloadCid) } - pieces, err := r.dagst.ShardsContainingMultihash(ctx, payloadCid.Hash()) + pieces, err := r.piecedirectory.PiecesContainingMultihash(ctx, payloadCid.Hash()) if err != nil { - if errors.Is(err, datastore.ErrNotFound) { + if types.IsNotFound(err) { return []string{}, nil } return nil, fmt.Errorf("getting shards containing cid %s: %w", payloadCid, err) @@ -114,18 +192,67 @@ func (r *resolver) PiecesWithRootPayloadCid(ctx context.Context, args struct{ Pa return pieceCids, nil } +func (r *resolver) PieceIndexes(ctx context.Context, args struct{ PieceCid string }) ([]string, error) { + var indexes []string + pieceCid, err := cid.Parse(args.PieceCid) + if err != nil { + return nil, fmt.Errorf("%s is not a valid piece cid", args.PieceCid) + } + + ii, err := r.piecedirectory.GetIterableIndex(ctx, pieceCid) + if err != nil { + return nil, fmt.Errorf("could not get indexes for %s: %w", pieceCid, err) + } + + err = ii.ForEach(func(m multihash.Multihash, _ uint64) error { + indexes = append(indexes, cid.NewCidV1(cid.DagProtobuf, m).String()) + return nil + }) + + if err != nil { + return nil, fmt.Errorf("iterating index for piece %s: %w", pieceCid, err) + } + + return indexes, nil +} + +func (r *resolver) PieceBuildIndex(args struct{ PieceCid string }) (bool, error) { + pieceCid, err := cid.Parse(args.PieceCid) + if err != nil { + return false, fmt.Errorf("%s is not a valid piece cid", args.PieceCid) + } + + // Use the global boost context for build piece, because if the user + // navigates away from the page we don't want to cancel the build piece + // operation + err = r.piecedirectory.BuildIndexForPiece(r.ctx, pieceCid) + if err != nil { + return false, err + } + return true, nil +} + func (r *resolver) PieceStatus(ctx context.Context, args struct{ PieceCid string }) (*pieceResolver, error) { pieceCid, err := cid.Parse(args.PieceCid) if err != nil { return nil, fmt.Errorf("%s is not a valid piece cid", args.PieceCid) } - // Get sector info from PieceStore - pieceInfo, err := r.ps.GetPieceInfo(pieceCid) - if err != nil && !errors.Is(err, retrievalmarket.ErrNotFound) { + allLegacyDeals, err := r.legacyProv.ListLocalDeals() + if err != nil { return nil, err } + return r.pieceStatus(ctx, pieceCid, allLegacyDeals) +} + +func (r *resolver) pieceStatus(ctx context.Context, pieceCid cid.Cid, allLegacyDeals []storagemarket.MinerDeal) (*pieceResolver, error) { + // Get piece info from local index directory + pieceInfo, pmErr := r.piecedirectory.GetPieceMetadata(ctx, pieceCid) + if pmErr != nil && !types.IsNotFound(pmErr) { + return nil, pmErr + } + // Get boost deals by piece Cid boostDeals, err := r.dealsDB.ByPieceCID(ctx, pieceCid) if err != nil { @@ -133,11 +260,6 @@ func (r *resolver) PieceStatus(ctx context.Context, args struct{ PieceCid string } // Get legacy markets deals by piece Cid - // TODO: add method to markets to filter deals by piece CID - allLegacyDeals, err := r.legacyProv.ListLocalDeals() - if err != nil { - return nil, err - } var legacyDeals []storagemarket.MinerDeal for _, dl := range allLegacyDeals { if dl.Ref.PieceCid != nil && *dl.Ref.PieceCid == pieceCid { @@ -145,27 +267,17 @@ func (r *resolver) PieceStatus(ctx context.Context, args struct{ PieceCid string } } - // Convert piece info deals to graphQL format + // Convert local index directory deals to graphQL format var pids []*pieceInfoDeal for _, dl := range pieceInfo.Deals { - // Check the sealing status of each deal - errMsg := "" - isUnsealed, err := r.sa.IsUnsealed(ctx, dl.SectorID, dl.Offset.Unpadded(), dl.Length.Unpadded()) - if err != nil { - errMsg = err.Error() - } - pids = append(pids, &pieceInfoDeal{ - SealStatus: &sealStatus{ - IsUnsealed: isUnsealed, - Error: errMsg, - }, - ChainDealID: gqltypes.Uint64(dl.DealID), + ChainDealID: gqltypes.Uint64(dl.ChainDealID), Sector: §orResolver{ ID: gqltypes.Uint64(dl.SectorID), - Offset: gqltypes.Uint64(dl.Offset), - Length: gqltypes.Uint64(dl.Length), + Offset: gqltypes.Uint64(dl.PieceOffset), + Length: gqltypes.Uint64(dl.PieceLength), }, + sa: r.sa, }) } @@ -185,24 +297,14 @@ func (r *resolver) PieceStatus(ctx context.Context, args struct{ PieceCid string } bd.Message = dl.Checkpoint.String() - // Only check the unseal state if the deal has already been added to a piece - st := &sealStatus{IsUnsealed: false} - if dl.Checkpoint >= dealcheckpoints.AddedPiece { - isUnsealed, err := r.sa.IsUnsealed(ctx, dl.SectorID, dl.Offset.Unpadded(), dl.Length.Unpadded()) - if err != nil { - st.Error = err.Error() - } - st.IsUnsealed = isUnsealed - } - deals = append(deals, &pieceDealResolver{ - Deal: &bd, - SealStatus: st, + Deal: &bd, Sector: §orResolver{ ID: gqltypes.Uint64(dl.SectorID), Offset: gqltypes.Uint64(dl.Offset), Length: gqltypes.Uint64(dl.Length), }, + sa: r.sa, }) } @@ -223,61 +325,50 @@ func (r *resolver) PieceStatus(ctx context.Context, args struct{ PieceCid string // For legacy deals the sector information is stored in the piece store sector := r.getLegacyDealSector(ctx, pids, dl.DealID) - - st := &sealStatus{IsUnsealed: false} if sector == nil { sector = §orResolver{ID: gqltypes.Uint64(dl.SectorNumber)} - } else { - secID := abi.SectorNumber(sector.ID) - offset := abi.PaddedPieceSize(sector.Offset).Unpadded() - len := abi.PaddedPieceSize(sector.Length).Unpadded() - isUnsealed, err := r.sa.IsUnsealed(ctx, secID, offset, len) - st = &sealStatus{IsUnsealed: isUnsealed} - if err != nil { - st.Error = err.Error() - } } deals = append(deals, &pieceDealResolver{ - Deal: &bd, - Sector: sector, - SealStatus: st, + Deal: &bd, + Sector: sector, + sa: r.sa, }) } - // Get the state of the piece in the DAG store - idxStatus, err := r.getIndexStatus(ctx, pieceCid, deals) + // Get the state of the piece's index + idxStatus, err := r.getIndexStatus(ctx, pieceCid, pieceInfo, pmErr, deals) if err != nil { return nil, err } return &pieceResolver{ - PieceCid: args.PieceCid, + PieceCid: pieceCid.String(), IndexStatus: idxStatus, PieceInfoDeals: pids, Deals: deals, }, nil } -func (r *resolver) getIndexStatus(ctx context.Context, pieceCid cid.Cid, deals []*pieceDealResolver) (*indexStatus, error) { - si, err := r.dagst.GetShardInfo(shard.KeyFromCID(pieceCid)) - if err != nil && !errors.Is(err, dagstore.ErrShardUnknown) { - return nil, err - } - idxst := IndexStatusUnknown +func (r *resolver) getIndexStatus(ctx context.Context, pieceCid cid.Cid, md pdtypes.PieceDirMetadata, mdErr error, deals []*pieceDealResolver) (*indexStatus, error) { + var idxst IndexStatus idxerr := "" + switch { - case err != nil && errors.Is(err, dagstore.ErrShardUnknown): + case mdErr != nil && types.IsNotFound(mdErr): idxst = IndexStatusNotFound - case si.ShardState == dagstore.ShardStateNew, si.ShardState == dagstore.ShardStateInitializing: + case mdErr != nil: + idxst = IndexStatusFailed + idxerr = mdErr.Error() + case md.Indexing: + idxst = IndexStatusIndexing + case md.Error != "": + idxst = IndexStatusFailed + idxerr = md.Error + case md.IndexedAt.IsZero(): idxst = IndexStatusRegistered - case si.ShardState == dagstore.ShardStateAvailable, si.ShardState == dagstore.ShardStateServing: + default: idxst = IndexStatusComplete - case si.ShardState == dagstore.ShardStateErrored, si.ShardState == dagstore.ShardStateRecovering: - idxst = IndexStatusFailed - if si.Error != nil { - idxerr = si.Error.Error() - } } // Try retrieving the piece payload cid as a means to check if the @@ -289,8 +380,8 @@ func (r *resolver) getIndexStatus(ctx context.Context, pieceCid cid.Cid, deals [ // This should never happen, but check just in case return nil, fmt.Errorf("parsing retrieved deal data root cid %s: %w", cidstr, err) } - ks, err := r.dagst.ShardsContainingMultihash(ctx, c.Hash()) - if err != nil || len(ks) == 0 { + pieces, err := r.piecedirectory.PiecesContainingMultihash(ctx, c.Hash()) + if err != nil || len(pieces) == 0 { idxst = IndexStatusFailed idxerr = fmt.Sprintf("unable to resolve piece's root payload cid %s to piece cid", cidstr) } @@ -330,3 +421,35 @@ func (r *resolver) getLegacyDealSector(ctx context.Context, pids []*pieceInfoDea } return nil } + +const isUnsealedTimeout = 5 * time.Second + +func sealStatus(ctx context.Context, sa retrievalmarket.SectorAccessor, sector *sectorResolver) *sealStatusResolver { + if sector == nil || sector.ID == 0 { + return &sealStatusResolver{Error: "unable to find sector for deal"} + } + if sector.Length == 0 { + return &sealStatusResolver{Error: fmt.Sprintf("sector %d has zero length", sector.ID)} + } + + ssr := &sealStatusResolver{} + isUnsealedCtx, cancel := context.WithTimeout(ctx, isUnsealedTimeout) + defer cancel() + + isUnsealed, err := sa.IsUnsealed( + isUnsealedCtx, abi.SectorNumber(sector.ID), + abi.PaddedPieceSize(sector.Offset).Unpadded(), + abi.PaddedPieceSize(sector.Length).Unpadded(), + ) + ssr.IsUnsealed = isUnsealed + if err != nil { + ssr.Error = err.Error() + if isUnsealedCtx.Err() != nil { + ssr.Error = fmt.Sprintf("IsUnsealed: timed out after %s "+ + "(IsUnsealed blocks if the sector is currently being unsealed)", + isUnsealedTimeout) + } + } + + return ssr +} diff --git a/gql/resolver_proplog.go b/gql/resolver_proplog.go index 3a25ba605..4f4192b40 100644 --- a/gql/resolver_proplog.go +++ b/gql/resolver_proplog.go @@ -2,8 +2,6 @@ package gql import ( "context" - "time" - "github.com/filecoin-project/boost/db" gqltypes "github.com/filecoin-project/boost/gql/types" "github.com/graph-gophers/graphql-go" @@ -56,12 +54,7 @@ func (r *resolver) ProposalLogs(ctx context.Context, args proposalLogsArgs) (*pr // Fetch one extra deal so that we can check if there are more deals // beyond the limit - var cursor *time.Time - if args.Cursor != nil { - val := (*args.Cursor).Int64() - asTime := time.Unix(val/1000, (val%1000)*1e6) - cursor = &asTime - } + cursor := bigIntToTime(args.Cursor) logs, err := r.plDB.List(ctx, args.Accepted.Value, cursor, offset, limit+1) if err != nil { return nil, err diff --git a/gql/schema.graphql b/gql/schema.graphql index 242d4be7f..a2bc3c9e0 100644 --- a/gql/schema.graphql +++ b/gql/schema.graphql @@ -183,6 +183,17 @@ type PieceStatus { PieceInfoDeals: [PieceInfoDeal]! } +type FlaggedPieceStatus { + CreatedAt: Time! + Piece: PieceStatus! +} + +type FlaggedPiecesList { + totalCount: Int! + pieces: [FlaggedPieceStatus]! + more: Boolean! +} + type ProposalLog { DealUUID: ID! Accepted: Boolean! @@ -439,6 +450,9 @@ type RootQuery { """Get the number of retrieval logs""" retrievalLogsCount: RetrievalStatesCount! + """Get a list of pieces that have been flagged as having problems""" + piecesFlagged(cursor: BigInt, offset: Int, limit: Int): FlaggedPiecesList! + """Get information about a piece from the piece store, DAG store and database""" pieceStatus(pieceCid: String!): PieceStatus! @@ -448,6 +462,9 @@ type RootQuery { """Get the pieces that have a particular root payload CID""" piecesWithRootPayloadCid(payloadCid: String!): [String!]! + """Get the indexes for a particular piece CID""" + pieceIndexes(pieceCid: String!): [String!]! + """Get storage space usage""" storage: Storage! @@ -495,6 +512,9 @@ type RootMutation { """Publish all pending deals now""" dealPublishNow: Boolean! + """Explicitly load the piece data from the sealing subsystem and index it""" + pieceBuildIndex(pieceCid: String!): Boolean! + """Top-up the available deal collateral in escrow for deal publishing""" fundsMoveToEscrow(amount: BigInt!): Boolean! diff --git a/gql/util.go b/gql/util.go new file mode 100644 index 000000000..35c9061ff --- /dev/null +++ b/gql/util.go @@ -0,0 +1,16 @@ +package gql + +import ( + "time" + + gqltypes "github.com/filecoin-project/boost/gql/types" +) + +func bigIntToTime(i *gqltypes.BigInt) *time.Time { + if i == nil { + return nil + } + val := (*i).Int64() + asTime := time.Unix(val/1000, (val%1000)*1e6) + return &asTime +} diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index dea5574d5..48d6852ad 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -4,60 +4,56 @@ import ( "context" "errors" "fmt" - "math" "os" "path/filepath" - "github.com/filecoin-project/lotus/node/repo" - "github.com/libp2p/go-libp2p/core/crypto" - host "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" - "go.uber.org/fx" - - "github.com/filecoin-project/boost/markets/idxprov" - dst "github.com/filecoin-project/dagstore" - "github.com/filecoin-project/lotus/markets/dagstore" - "github.com/filecoin-project/boost/db" + "github.com/filecoin-project/boost/markets/idxprov" "github.com/filecoin-project/boost/node/config" + "github.com/filecoin-project/boost/piecedirectory" + "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints" lotus_storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/lotus/node/repo" "github.com/hashicorp/go-multierror" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - - "github.com/filecoin-project/boost/storagemarket/types" + provider "github.com/ipni/index-provider" "github.com/ipni/index-provider/engine/xproviders" "github.com/ipni/index-provider/metadata" - - "github.com/ipfs/go-cid" - provider "github.com/ipni/index-provider" + "github.com/libp2p/go-libp2p/core/crypto" + host "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multihash" + "go.uber.org/fx" ) var log = logging.Logger("index-provider-wrapper") -var shardRegMarker = ".boost-shard-registration-complete" var defaultDagStoreDir = "dagstore" type Wrapper struct { - cfg *config.Boost - enabled bool - dealsDB *db.DealsDB - legacyProv lotus_storagemarket.StorageProvider - prov provider.Interface - dagStore *dagstore.Wrapper - meshCreator idxprov.MeshCreator - h host.Host + cfg *config.Boost + enabled bool + dealsDB *db.DealsDB + legacyProv lotus_storagemarket.StorageProvider + prov provider.Interface + piecedirectory *piecedirectory.PieceDirectory + meshCreator idxprov.MeshCreator + h host.Host // bitswapEnabled records whether to announce bitswap as an available // protocol to the network indexer bitswapEnabled bool } func NewWrapper(cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dealsDB *db.DealsDB, - legacyProv lotus_storagemarket.StorageProvider, prov provider.Interface, dagStore *dagstore.Wrapper, - meshCreator idxprov.MeshCreator) (*Wrapper, error) { + legacyProv lotus_storagemarket.StorageProvider, prov provider.Interface, + piecedirectory *piecedirectory.PieceDirectory, meshCreator idxprov.MeshCreator) (*Wrapper, error) { return func(lc fx.Lifecycle, h host.Host, r repo.LockedRepo, dealsDB *db.DealsDB, - legacyProv lotus_storagemarket.StorageProvider, prov provider.Interface, dagStore *dagstore.Wrapper, + legacyProv lotus_storagemarket.StorageProvider, prov provider.Interface, + piecedirectory *piecedirectory.PieceDirectory, meshCreator idxprov.MeshCreator) (*Wrapper, error) { + if cfg.DAGStore.RootDir == "" { cfg.DAGStore.RootDir = filepath.Join(r.Path(), defaultDagStoreDir) } @@ -73,11 +69,11 @@ func NewWrapper(cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, r repo.Loc dealsDB: dealsDB, legacyProv: legacyProv, prov: prov, - dagStore: dagStore, meshCreator: meshCreator, cfg: cfg, - bitswapEnabled: bitswapEnabled, enabled: !isDisabled, + piecedirectory: piecedirectory, + bitswapEnabled: bitswapEnabled, } // announce all deals on startup in case of a config change lc.Append(fx.Hook{ @@ -271,47 +267,53 @@ func (w *Wrapper) IndexerAnnounceAllDeals(ctx context.Context) error { } func (w *Wrapper) Start(ctx context.Context) { - // re-init dagstore shards for Boost deals if needed - if _, err := w.DagstoreReinitBoostDeals(ctx); err != nil { - log.Errorw("failed to migrate dagstore indices for Boost deals", "err", err) - } + w.prov.RegisterMultihashLister(w.MultihashLister) +} - w.prov.RegisterMultihashLister(func(ctx context.Context, pid peer.ID, contextID []byte) (provider.MultihashIterator, error) { - provideF := func(pieceCid cid.Cid) (provider.MultihashIterator, error) { - ii, err := w.dagStore.GetIterableIndexForPiece(pieceCid) - if err != nil { - return nil, fmt.Errorf("failed to get iterable index: %w", err) - } +func (w *Wrapper) MultihashLister(ctx context.Context, prov peer.ID, contextID []byte) (provider.MultihashIterator, error) { + provideF := func(pieceCid cid.Cid) (provider.MultihashIterator, error) { + ii, err := w.piecedirectory.GetIterableIndex(ctx, pieceCid) + if err != nil { + return nil, fmt.Errorf("failed to get iterable index: %w", err) + } - mhi, err := provider.CarMultihashIterator(ii) - if err != nil { - return nil, fmt.Errorf("failed to get mhiterator: %w", err) - } - return mhi, nil + // Check if there are any records in the iterator. If there are no + // records, the multihash lister expects us to return an error. + hasRecords := ii.ForEach(func(_ multihash.Multihash, _ uint64) error { + return fmt.Errorf("has at least one record") + }) + if hasRecords == nil { + return nil, fmt.Errorf("no records found for piece %s", pieceCid) } - // convert context ID to proposal Cid - proposalCid, err := cid.Cast(contextID) + mhi, err := provider.CarMultihashIterator(ii) if err != nil { - return nil, fmt.Errorf("failed to cast context ID to a cid") + return nil, fmt.Errorf("failed to get mhiterator: %w", err) } + return mhi, nil + } - // go from proposal cid -> piece cid by looking up deal in boost and if we can't find it there -> then markets - // check Boost deals DB - pds, boostErr := w.dealsDB.BySignedProposalCID(ctx, proposalCid) - if boostErr == nil { - pieceCid := pds.ClientDealProposal.Proposal.PieceCID - return provideF(pieceCid) - } + // convert context ID to proposal Cid + proposalCid, err := cid.Cast(contextID) + if err != nil { + return nil, fmt.Errorf("failed to cast context ID to a cid") + } - // check in legacy markets - md, legacyErr := w.legacyProv.GetLocalDeal(proposalCid) - if legacyErr == nil { - return provideF(md.Proposal.PieceCID) - } + // go from proposal cid -> piece cid by looking up deal in boost and if we can't find it there -> then markets + // check Boost deals DB + pds, boostErr := w.dealsDB.BySignedProposalCID(ctx, proposalCid) + if boostErr == nil { + pieceCid := pds.ClientDealProposal.Proposal.PieceCID + return provideF(pieceCid) + } - return nil, fmt.Errorf("failed to look up deal in Boost, err=%s and Legacy Markets, err=%s", boostErr, legacyErr) - }) + // check in legacy markets + md, legacyErr := w.legacyProv.GetLocalDeal(proposalCid) + if legacyErr == nil { + return provideF(md.Proposal.PieceCID) + } + + return nil, fmt.Errorf("failed to look up deal in Boost, err=%s and Legacy Markets, err=%s", boostErr, legacyErr) } func (w *Wrapper) AnnounceBoostDeal(ctx context.Context, pds *types.ProviderDealState) (cid.Cid, error) { @@ -347,130 +349,3 @@ func (w *Wrapper) AnnounceBoostDeal(ctx context.Context, pds *types.ProviderDeal } return annCid, err } - -func (w *Wrapper) DagstoreReinitBoostDeals(ctx context.Context) (bool, error) { - deals, err := w.dealsDB.ListActive(ctx) - if err != nil { - return false, fmt.Errorf("failed to list active Boost deals: %w", err) - } - - log := log.Named("boost-migrator") - log.Infof("dagstore root is %s", w.cfg.DAGStore.RootDir) - - // Check if all deals have already been registered as shards - isComplete, err := w.boostRegistrationComplete() - if err != nil { - return false, fmt.Errorf("failed to get boost dagstore migration status: %w", err) - } - if isComplete { - // All deals have been registered as shards, bail out - log.Info("no boost shard migration necessary; already marked complete") - return false, nil - } - - log.Infow("registering shards for all active boost deals in sealing subsystem", "count", len(deals)) - - // channel where results will be received, and channel where the total - // number of registered shards will be sent. - resch := make(chan dst.ShardResult, 32) - totalCh := make(chan int) - doneCh := make(chan struct{}) - - // Start making progress consuming results. We won't know how many to - // actually consume until we register all shards. - // - // If there are any problems registering shards, just log an error - go func() { - defer close(doneCh) - - var total = math.MaxInt64 - var res dst.ShardResult - for rcvd := 0; rcvd < total; { - select { - case total = <-totalCh: - // we now know the total number of registered shards - // nullify so that we no longer consume from it after closed. - close(totalCh) - totalCh = nil - case res = <-resch: - rcvd++ - if res.Error == nil { - log.Infow("async boost shard registration completed successfully", "shard_key", res.Key) - } else { - log.Warnw("async boost shard registration failed", "shard_key", res.Key, "error", res.Error) - } - } - } - }() - - var registered int - for _, deal := range deals { - pieceCid := deal.ClientDealProposal.Proposal.PieceCID - - // enrich log statements in this iteration with deal ID and piece CID. - log := log.With("deal_id", deal.ChainDealID, "piece_cid", pieceCid) - - // Filter out deals that have not yet been indexed and announced as they will be re-indexed anyways - if deal.Checkpoint < dealcheckpoints.IndexedAndAnnounced { - continue - } - - log.Infow("registering boost deal in dagstore with lazy init") - - // Register the deal as a shard with the DAG store with lazy initialization. - // The index will be populated the first time the deal is retrieved, or - // through the bulk initialization script. - err = w.dagStore.RegisterShard(ctx, pieceCid, "", false, resch) - if err != nil { - log.Warnw("failed to register boost shard", "error", err) - continue - } - registered++ - } - - log.Infow("finished registering all boost shards", "total", registered) - totalCh <- registered - select { - case <-ctx.Done(): - return false, ctx.Err() - case <-doneCh: - } - - log.Infow("confirmed registration of all boost shards") - - // Completed registering all shards, so mark the migration as complete - err = w.markBoostRegistrationComplete() - if err != nil { - log.Errorf("failed to mark boost shards as registered: %s", err) - } else { - log.Info("successfully marked boost migration as complete") - } - - log.Infow("boost dagstore migration complete") - - return true, nil -} - -// Check for the existence of a "marker" file indicating that the migration -// has completed -func (w *Wrapper) boostRegistrationComplete() (bool, error) { - path := filepath.Join(w.cfg.DAGStore.RootDir, shardRegMarker) - _, err := os.Stat(path) - if os.IsNotExist(err) { - return false, nil - } - if err != nil { - return false, err - } - return true, nil -} - -// Create a "marker" file indicating that the migration has completed -func (w *Wrapper) markBoostRegistrationComplete() error { - path := filepath.Join(w.cfg.DAGStore.RootDir, shardRegMarker) - file, err := os.Create(path) - if err != nil { - return err - } - return file.Close() -} diff --git a/markets/storageadapter/api.go b/markets/storageadapter/api.go index 0bd75e790..a3192492d 100644 --- a/markets/storageadapter/api.go +++ b/markets/storageadapter/api.go @@ -3,17 +3,15 @@ package storageadapter import ( "context" - "github.com/ipfs/go-cid" - cbor "github.com/ipfs/go-ipld-cbor" - "github.com/ipfs/go-libipfs/blocks" - "golang.org/x/xerrors" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "github.com/ipfs/go-libipfs/blocks" + "golang.org/x/xerrors" ) type apiWrapper struct { diff --git a/markets/storageadapter/ondealsectorcommitted_test.go b/markets/storageadapter/ondealsectorcommitted_test.go index 76f7a35ac..5433d6480 100644 --- a/markets/storageadapter/ondealsectorcommitted_test.go +++ b/markets/storageadapter/ondealsectorcommitted_test.go @@ -10,19 +10,12 @@ import ( "testing" "time" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/blocks" - "github.com/stretchr/testify/require" - "golang.org/x/xerrors" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin" markettypes "github.com/filecoin-project/go-state-types/builtin/v9/market" minertypes "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/go-state-types/cbor" - tutils "github.com/filecoin-project/specs-actors/v2/support/testing" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" @@ -30,6 +23,11 @@ import ( test "github.com/filecoin-project/lotus/chain/events/state/mock" "github.com/filecoin-project/lotus/chain/types" pipeline "github.com/filecoin-project/lotus/storage/pipeline" + tutils "github.com/filecoin-project/specs-actors/v2/support/testing" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" ) func TestOnDealSectorPreCommitted(t *testing.T) { diff --git a/metrics/metrics.go b/metrics/metrics.go index 7fa9b6707..394908ea8 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -128,6 +128,12 @@ var ( HttpPieceByCid400ResponseCount = stats.Int64("http/piece_by_cid_400_response_count", "Counter of /piece/ 400 responses", stats.UnitDimensionless) HttpPieceByCid404ResponseCount = stats.Int64("http/piece_by_cid_404_response_count", "Counter of /piece/ 404 responses", stats.UnitDimensionless) HttpPieceByCid500ResponseCount = stats.Int64("http/piece_by_cid_500_response_count", "Counter of /piece/ 500 responses", stats.UnitDimensionless) + HttpBlockByCidRequestCount = stats.Int64("http/block_by_cid_request_count", "Counter of /block/ requests", stats.UnitDimensionless) + HttpBlockByCidRequestDuration = stats.Float64("http/block_by_cid_request_duration_ms", "Time spent retrieving a block by cid", stats.UnitMilliseconds) + HttpBlockByCid200ResponseCount = stats.Int64("http/block_by_cid_200_response_count", "Counter of /block/ 200 responses", stats.UnitDimensionless) + HttpBlockByCid400ResponseCount = stats.Int64("http/block_by_cid_400_response_count", "Counter of /block/ 400 responses", stats.UnitDimensionless) + HttpBlockByCid404ResponseCount = stats.Int64("http/block_by_cid_404_response_count", "Counter of /block/ 404 responses", stats.UnitDimensionless) + HttpBlockByCid500ResponseCount = stats.Int64("http/block_by_cid_500_response_count", "Counter of /block/ 500 responses", stats.UnitDimensionless) // bitswap BitswapRblsGetRequestCount = stats.Int64("bitswap/rbls_get_request_count", "Counter of RemoteBlockstore Get requests", stats.UnitDimensionless) diff --git a/node/builder.go b/node/builder.go index b94fb63a3..5801e614e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -24,6 +24,8 @@ import ( "github.com/filecoin-project/boost/node/modules" "github.com/filecoin-project/boost/node/modules/dtypes" "github.com/filecoin-project/boost/node/repo" + "github.com/filecoin-project/boost/piecedirectory" + pdtypes "github.com/filecoin-project/boost/piecedirectory/types" "github.com/filecoin-project/boost/protocolproxy" "github.com/filecoin-project/boost/retrievalmarket/lp2pimpl" "github.com/filecoin-project/boost/retrievalmarket/rtvllog" @@ -40,7 +42,6 @@ import ( rmnet "github.com/filecoin-project/go-fil-markets/retrievalmarket/network" lotus_storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/storedask" - "github.com/filecoin-project/go-fil-markets/stores" "github.com/filecoin-project/go-state-types/abi" lotus_api "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" @@ -79,10 +80,12 @@ import ( "github.com/libp2p/go-libp2p/p2p/net/conngater" "github.com/multiformats/go-multiaddr" "go.uber.org/fx" + "go.uber.org/fx/fxevent" ) //nolint:deadcode,varcheck var log = logging.Logger("builder") +var fxlog = logging.Logger("fxlog") // special is a type used to give keys to modules which // @@ -144,6 +147,7 @@ const ( // miner GetParamsKey + StartPieceDoctorKey HandleMigrateProviderFundsKey HandleDealsKey HandleCreateRetrievalTablesKey @@ -394,7 +398,9 @@ func New(ctx context.Context, opts ...Option) (StopFunc, error) { fx.Options(ctors...), fx.Options(settings.invokes...), - fx.NopLogger, + fx.WithLogger(func() fxevent.Logger { + return &fxevent.ZapLogger{Logger: fxlog.Desugar()} + }), ) // TODO: we probably should have a 'firewall' for Closing signal @@ -521,7 +527,8 @@ func ConfigBoost(cfg *config.Boost) Option { Override(new(server.AskGetter), From(new(*modules.ProxyAskGetter))), Override(new(*server.GraphsyncUnpaidRetrieval), modules.Graphsync(cfg.LotusDealmaking.SimultaneousTransfersForStorage, cfg.LotusDealmaking.SimultaneousTransfersForStoragePerClient, cfg.LotusDealmaking.SimultaneousTransfersForRetrieval)), Override(new(lotus_dtypes.StagingGraphsync), From(new(*server.GraphsyncUnpaidRetrieval))), - Override(new(lotus_dtypes.ProviderPieceStore), lotus_modules.NewProviderPieceStore), + Override(new(lotus_dtypes.ProviderPieceStore), modules.NewProviderPieceStore), + Override(StartPieceDoctorKey, modules.NewPieceDoctor), // Lotus Markets (retrieval deps) Override(new(sealer.PieceProvider), sealer.NewPieceProvider), @@ -534,10 +541,19 @@ func ConfigBoost(cfg *config.Boost) Option { })), // DAG Store - Override(new(mdagstore.MinerAPI), lotus_modules.NewMinerAPI(cfg.DAGStore)), - Override(DAGStoreKey, lotus_modules.DAGStore(cfg.DAGStore)), + //Override(new(dagstore.MinerAPI), lotus_modules.NewMinerAPI(cfg.DAGStore)), + + // TODO: Not sure how to completely get rid of these yet: + // Error: creating node: starting node: missing dependencies for function "reflect".makeFuncStub (/usr/local/go/src/reflect/asm_amd64.s:30): missing types: *dagstore.DAGStore; *dagstore.Wrapper (did you mean stores.DAGStoreWrapper?) + Override(new(*dagstore.DAGStore), func() *dagstore.DAGStore { return nil }), + Override(new(*mdagstore.Wrapper), func() *mdagstore.Wrapper { return nil }), + + Override(new(pdtypes.Store), modules.NewPieceDirectoryStore(cfg)), + Override(new(*piecedirectory.PieceDirectory), modules.NewPieceDirectory(cfg)), + Override(DAGStoreKey, modules.NewDAGStoreWrapper), + //Override(new(mktsdagstore.MinerAPI), lotus_modules.NewMinerAPI(cfg.DAGStore)), + //Override(DAGStoreKey, lotus_modules.DAGStore(cfg.DAGStore)), Override(new(dagstore.Interface), From(new(*dagstore.DAGStore))), - Override(new(stores.DAGStoreWrapper), From(new(*mdagstore.Wrapper))), Override(new(*modules.ShardSelector), modules.NewShardSelector), Override(new(dtypes.IndexBackedBlockstore), modules.NewIndexBackedBlockstore(cfg)), Override(HandleSetShardSelector, modules.SetShardSelectorFunc), @@ -547,7 +563,7 @@ func ConfigBoost(cfg *config.Boost) Option { Override(new(retrievalmarket.SectorAccessor), From(new(mdagstore.SectorAccessor))), Override(new(retrievalmarket.RetrievalProviderNode), retrievaladapter.NewRetrievalProviderNode), Override(new(rmnet.RetrievalMarketNetwork), lotus_modules.RetrievalNetwork), - Override(new(retrievalmarket.RetrievalProvider), lotus_modules.RetrievalProvider), + Override(new(retrievalmarket.RetrievalProvider), modules.RetrievalProvider), Override(HandleSetRetrievalAskGetter, modules.SetAskGetter), Override(HandleRetrievalEventsKey, modules.HandleRetrievalGraphsyncUpdates(time.Duration(cfg.Dealmaking.RetrievalLogDuration), time.Duration(cfg.Dealmaking.StalledRetrievalTimeout))), Override(HandleRetrievalKey, lotus_modules.HandleRetrieval), diff --git a/node/config/def.go b/node/config/def.go index 1cd5bbc6b..93511fafc 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -72,6 +72,20 @@ func DefaultBoost() *Boost { ServiceName: "boostd", }, + LocalIndexDirectory: LocalIndexDirectoryConfig{ + Couchbase: LocalIndexDirectoryCouchbaseConfig{ + ConnectString: "", + Username: "", + Password: "", + PieceMetadataBucket: LocalIndexDirectoryCouchbaseBucketConfig{}, + MultihashToPiecesBucket: LocalIndexDirectoryCouchbaseBucketConfig{}, + PieceOffsetsBucket: LocalIndexDirectoryCouchbaseBucketConfig{}, + }, + ParallelAddIndexLimit: 4, + EmbeddedServicePort: 8042, + ServiceApiInfo: "", + }, + ContractDeals: ContractDealsConfig{ Enabled: false, AllowlistContracts: []string{}, diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index c99d7bcfb..d33ff44c0 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -67,6 +67,12 @@ your node if metadata log is disabled`, Comment: ``, }, + { + Name: "LocalIndexDirectory", + Type: "LocalIndexDirectoryConfig", + + Comment: ``, + }, { Name: "ContractDeals", Type: "ContractDealsConfig", @@ -422,6 +428,85 @@ Any value less than 0 will result in use of default`, Comment: `The port that the graphql server listens on`, }, }, + "LocalIndexDirectoryConfig": []DocField{ + { + Name: "Couchbase", + Type: "LocalIndexDirectoryCouchbaseConfig", + + Comment: ``, + }, + { + Name: "ParallelAddIndexLimit", + Type: "int", + + Comment: `The maximum number of add index operations allowed to execute in parallel. +The add index operation is executed when a new deal is created - it fetches +the piece from the sealing subsystem, creates an index of where each block +is in the piece, and adds the index to the local index directory.`, + }, + { + Name: "EmbeddedServicePort", + Type: "uint64", + + Comment: `The port that the embedded local index directory data service runs on. +Set this value to zero to disable the embedded local index directory data service +(in that case the local index directory data service must be running externally)`, + }, + { + Name: "ServiceApiInfo", + Type: "string", + + Comment: `The connect string for the local index directory data service RPC API eg "http://localhost:8042" +Set this value to "" if the local index directory data service is embedded.`, + }, + }, + "LocalIndexDirectoryCouchbaseBucketConfig": []DocField{ + { + Name: "RAMQuotaMB", + Type: "uint64", + + Comment: `Bucket setting RAMQuotaMB`, + }, + }, + "LocalIndexDirectoryCouchbaseConfig": []DocField{ + { + Name: "ConnectString", + Type: "string", + + Comment: `The couchbase connect string eg "couchbase://127.0.0.1" +If empty, a leveldb database is used instead.`, + }, + { + Name: "Username", + Type: "string", + + Comment: ``, + }, + { + Name: "Password", + Type: "string", + + Comment: ``, + }, + { + Name: "PieceMetadataBucket", + Type: "LocalIndexDirectoryCouchbaseBucketConfig", + + Comment: ``, + }, + { + Name: "MultihashToPiecesBucket", + Type: "LocalIndexDirectoryCouchbaseBucketConfig", + + Comment: ``, + }, + { + Name: "PieceOffsetsBucket", + Type: "LocalIndexDirectoryCouchbaseBucketConfig", + + Comment: ``, + }, + }, "LotusDealmakingConfig": []DocField{ { Name: "PieceCidBlocklist", diff --git a/node/config/types.go b/node/config/types.go index 631ec577f..f524e07fe 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -38,12 +38,13 @@ type Boost struct { // The connect string for the sealing RPC API (lotus miner) SealerApiInfo string // The connect string for the sector index RPC API (lotus miner) - SectorIndexApiInfo string - Dealmaking DealmakingConfig - Wallets WalletsConfig - Graphql GraphqlConfig - Tracing TracingConfig - ContractDeals ContractDealsConfig + SectorIndexApiInfo string + Dealmaking DealmakingConfig + Wallets WalletsConfig + Graphql GraphqlConfig + Tracing TracingConfig + LocalIndexDirectory LocalIndexDirectoryConfig + ContractDeals ContractDealsConfig // Lotus configs LotusDealmaking lotus_config.DealmakingConfig @@ -290,3 +291,35 @@ type StorageConfig struct { // The maximum number of concurrent fetch operations to the storage subsystem ParallelFetchLimit int } + +type LocalIndexDirectoryCouchbaseBucketConfig struct { + // Bucket setting RAMQuotaMB + RAMQuotaMB uint64 +} + +type LocalIndexDirectoryCouchbaseConfig struct { + // The couchbase connect string eg "couchbase://127.0.0.1" + // If empty, a leveldb database is used instead. + ConnectString string + Username string + Password string + PieceMetadataBucket LocalIndexDirectoryCouchbaseBucketConfig + MultihashToPiecesBucket LocalIndexDirectoryCouchbaseBucketConfig + PieceOffsetsBucket LocalIndexDirectoryCouchbaseBucketConfig +} + +type LocalIndexDirectoryConfig struct { + Couchbase LocalIndexDirectoryCouchbaseConfig + // The maximum number of add index operations allowed to execute in parallel. + // The add index operation is executed when a new deal is created - it fetches + // the piece from the sealing subsystem, creates an index of where each block + // is in the piece, and adds the index to the local index directory. + ParallelAddIndexLimit int + // The port that the embedded local index directory data service runs on. + // Set this value to zero to disable the embedded local index directory data service + // (in that case the local index directory data service must be running externally) + EmbeddedServicePort uint64 + // The connect string for the local index directory data service RPC API eg "http://localhost:8042" + // Set this value to "" if the local index directory data service is embedded. + ServiceApiInfo string +} diff --git a/node/impl/boost.go b/node/impl/boost.go index 9b1ecd9bd..a3dbc0ec8 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -5,10 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "sort" "github.com/filecoin-project/boost/node/impl/backupmgr" + "github.com/filecoin-project/boost/piecedirectory" "github.com/multiformats/go-multihash" "go.opentelemetry.io/otel/attribute" @@ -79,6 +81,9 @@ type BoostAPI struct { // Sealing Pipeline API Sps sealingpipeline.API + // Piece Directory + Pd *piecedirectory.PieceDirectory + // GraphSQL server GraphqlServer *gql.Server @@ -147,6 +152,28 @@ func (sm *BoostAPI) BoostIndexerAnnounceAllDeals(ctx context.Context) error { return sm.IndexProvider.IndexerAnnounceAllDeals(ctx) } +// BoostIndexerListMultihashes calls the index provider multihash lister for a given proposal cid +func (sm *BoostAPI) BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error) { + it, err := sm.IndexProvider.MultihashLister(ctx, "", proposalCid.Bytes()) + if err != nil { + return nil, err + } + + var mhs []multihash.Multihash + mh, err := it.Next() + for { + if err != nil { + if errors.Is(err, io.EOF) { + return mhs, nil + } + return nil, err + } + mhs = append(mhs, mh) + + mh, err = it.Next() + } +} + func (sm *BoostAPI) BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string) (*api.ProviderDealRejectionInfo, error) { res, err := sm.StorageProvider.ImportOfflineDealData(ctx, dealUuid, filePath) return res, err @@ -499,19 +526,27 @@ func (sm *BoostAPI) BoostMakeDeal(ctx context.Context, params types.DealParams) } func (sm *BoostAPI) BlockstoreGet(ctx context.Context, c cid.Cid) ([]byte, error) { - blk, err := sm.IndexBackedBlockstore.Get(ctx, c) - if err != nil { - return nil, err - } - return blk.RawData(), nil + return sm.Pd.BlockstoreGet(ctx, c) } func (sm *BoostAPI) BlockstoreHas(ctx context.Context, c cid.Cid) (bool, error) { - return sm.IndexBackedBlockstore.Has(ctx, c) + return sm.Pd.BlockstoreHas(ctx, c) } func (sm *BoostAPI) BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) { - return sm.IndexBackedBlockstore.GetSize(ctx, c) + return sm.Pd.BlockstoreGetSize(ctx, c) +} + +func (sm *BoostAPI) PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "Boost.PdBuildIndexForPieceCid") + span.SetAttributes(attribute.String("piececid", piececid.String())) + defer span.End() + + return sm.Pd.BuildIndexForPiece(ctx, piececid) +} + +func (sm *BoostAPI) PdMarkIndexErrored(ctx context.Context, piececid cid.Cid, err string) error { + return sm.Pd.MarkIndexErrored(ctx, piececid, err) } func (sm *BoostAPI) OnlineBackup(ctx context.Context, dstDir string) error { diff --git a/node/impl/boost_legacy.go b/node/impl/boost_legacy.go index 88e309348..b6eb88b34 100644 --- a/node/impl/boost_legacy.go +++ b/node/impl/boost_legacy.go @@ -7,12 +7,8 @@ import ( "strconv" "time" - "github.com/filecoin-project/dagstore/shard" - "github.com/multiformats/go-multihash" - "github.com/filecoin-project/go-address" datatransfer "github.com/filecoin-project/go-data-transfer" - "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" @@ -217,52 +213,6 @@ func (sm *BoostAPI) ActorSectorSize(ctx context.Context, addr address.Address) ( return mi.SectorSize, nil } -func (sm *BoostAPI) PiecesListPieces(ctx context.Context) ([]cid.Cid, error) { - return sm.PieceStore.ListPieceInfoKeys() -} - -func (sm *BoostAPI) PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error) { - return sm.PieceStore.ListCidInfoKeys() -} - -func (sm *BoostAPI) PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) { - pi, err := sm.PieceStore.GetPieceInfo(pieceCid) - if err != nil { - return nil, fmt.Errorf("getting piece from piece store: %w", err) - } - return &pi, nil -} - -func (sm *BoostAPI) PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error) { - ci, err := sm.PieceStore.GetCIDInfo(payloadCid) - if err != nil { - return nil, err - } - - return &ci, nil -} - -func (sm *BoostAPI) PiecesGetMaxOffset(ctx context.Context, pieceCid cid.Cid) (uint64, error) { - var maxOffset uint64 - - it, err := sm.DAGStore.GetIterableIndex(shard.KeyFromCID(pieceCid)) - if err != nil { - return maxOffset, fmt.Errorf("getting iterable index for piece %s from DAG store: %w", pieceCid, err) - } - - err = it.ForEach(func(mh multihash.Multihash, offset uint64) error { - if offset > maxOffset { - maxOffset = offset - } - return nil - }) - if err != nil { - return maxOffset, fmt.Errorf("iterating over CAR index: %w", err) - } - - return maxOffset, err -} - func (sm *BoostAPI) RuntimeSubsystems(context.Context) (res lapi.MinerSubsystems, err error) { return []lapi.MinerSubsystem{lapi.SubsystemMarkets}, nil } diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index fc4f72fcc..f0d6abc04 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -84,5 +84,4 @@ type ProviderPieceStore piecestore.PieceStore type ProviderRequestValidator *requestvalidation.UnifiedRequestValidator -type StagingBlockstore blockstore.BasicBlockstore type StagingGraphsync graphsync.GraphExchange diff --git a/node/modules/graphsync.go b/node/modules/graphsync.go index 777b8afd2..4c6c60a93 100644 --- a/node/modules/graphsync.go +++ b/node/modules/graphsync.go @@ -2,11 +2,12 @@ package modules import ( "context" - "github.com/filecoin-project/boost/node/modules/dtypes" + + "github.com/filecoin-project/boost/cmd/booster-bitswap/remoteblockstore" + "github.com/filecoin-project/boost/piecedirectory" "github.com/filecoin-project/boost/retrievalmarket/server" "github.com/filecoin-project/go-fil-markets/retrievalmarket" retrievalimpl "github.com/filecoin-project/go-fil-markets/retrievalmarket/impl" - "github.com/filecoin-project/go-fil-markets/stores" "github.com/filecoin-project/go-state-types/abi" lotus_modules "github.com/filecoin-project/lotus/node/modules" lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" @@ -43,16 +44,18 @@ func SetAskGetter(proxy *ProxyAskGetter, rp retrievalmarket.RetrievalProvider) { } // Graphsync creates a graphsync instance used to serve retrievals. -func Graphsync(parallelTransfersForStorage uint64, parallelTransfersForStoragePerPeer uint64, parallelTransfersForRetrieval uint64) func(mctx lotus_helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.IndexBackedBlockstore, h host.Host, net lotus_dtypes.ProviderTransferNetwork, dealDecider lotus_dtypes.RetrievalDealFilter, dagStore stores.DAGStoreWrapper, pstore lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, askGetter server.AskGetter) (*server.GraphsyncUnpaidRetrieval, error) { - return func(mctx lotus_helpers.MetricsCtx, lc fx.Lifecycle, ibs dtypes.IndexBackedBlockstore, h host.Host, net lotus_dtypes.ProviderTransferNetwork, dealDecider lotus_dtypes.RetrievalDealFilter, dagStore stores.DAGStoreWrapper, pstore lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, askGetter server.AskGetter) (*server.GraphsyncUnpaidRetrieval, error) { +func Graphsync(parallelTransfersForStorage uint64, parallelTransfersForStoragePerPeer uint64, parallelTransfersForRetrieval uint64) func(mctx lotus_helpers.MetricsCtx, lc fx.Lifecycle, pid *piecedirectory.PieceDirectory, h host.Host, net lotus_dtypes.ProviderTransferNetwork, dealDecider lotus_dtypes.RetrievalDealFilter, pstore lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, askGetter server.AskGetter) (*server.GraphsyncUnpaidRetrieval, error) { + return func(mctx lotus_helpers.MetricsCtx, lc fx.Lifecycle, pid *piecedirectory.PieceDirectory, h host.Host, net lotus_dtypes.ProviderTransferNetwork, dealDecider lotus_dtypes.RetrievalDealFilter, pstore lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, askGetter server.AskGetter) (*server.GraphsyncUnpaidRetrieval, error) { + rb := remoteblockstore.NewRemoteBlockstore(pid) + // Create a Graphsync instance mkgs := lotus_modules.StagingGraphsync(parallelTransfersForStorage, parallelTransfersForStoragePerPeer, parallelTransfersForRetrieval) - gs := mkgs(mctx, lc, ibs, h) + gs := mkgs(mctx, lc, rb, h) // Wrap the Graphsync instance with a handler for unpaid retrieval requests vdeps := server.ValidationDeps{ DealDecider: retrievalimpl.DealDecider(dealDecider), - DagStore: dagStore, + DagStore: NewDAGStoreWrapper(pid), PieceStore: pstore, SectorAccessor: sa, AskStore: askGetter, diff --git a/node/modules/piecedirectory.go b/node/modules/piecedirectory.go new file mode 100644 index 000000000..8b6874962 --- /dev/null +++ b/node/modules/piecedirectory.go @@ -0,0 +1,285 @@ +package modules + +import ( + "context" + "fmt" + + "github.com/filecoin-project/boost/markets/sectoraccessor" + "github.com/filecoin-project/boost/node/config" + "github.com/filecoin-project/boost/piecedirectory" + "github.com/filecoin-project/boost/piecedirectory/types" + "github.com/filecoin-project/boostd-data/couchbase" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/dagstore" + "github.com/filecoin-project/dagstore/shard" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-fil-markets/piecestore" + "github.com/filecoin-project/go-fil-markets/shared" + "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-fil-markets/stores" + "github.com/filecoin-project/lotus/api/v1api" + mktsdagstore "github.com/filecoin-project/lotus/markets/dagstore" + "github.com/filecoin-project/lotus/node/modules/dtypes" + lotus_repo "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/filecoin-project/lotus/storage/sectorblocks" + "github.com/ipfs/go-cid" + bstore "github.com/ipfs/go-ipfs-blockstore" + carindex "github.com/ipld/go-car/v2/index" + "go.uber.org/fx" +) + +func NewPieceDirectoryStore(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_repo.LockedRepo) types.Store { + return func(lc fx.Lifecycle, r lotus_repo.LockedRepo) types.Store { + client := piecedirectory.NewStore() + + var cancel context.CancelFunc + var svcCtx context.Context + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + if cfg.LocalIndexDirectory.ServiceApiInfo != "" { + return client.Dial(ctx, cfg.LocalIndexDirectory.ServiceApiInfo) + } + + port := int(cfg.LocalIndexDirectory.EmbeddedServicePort) + if port == 0 { + return fmt.Errorf("starting local index directory client:" + + "either LocalIndexDirectory.ServiceApiInfo must be defined or " + + "LocalIndexDirectory.EmbeddedServicePort must be non-zero") + } + + svcCtx, cancel = context.WithCancel(ctx) + + var bdsvc *svc.Service + if cfg.LocalIndexDirectory.Couchbase.ConnectString != "" { + log.Infow("local index directory: connecting to couchbase server", + "connect-string", cfg.LocalIndexDirectory.Couchbase.ConnectString) + + // If the couchbase connect string is defined, set up a + // local index directory service that connects to the couchbase db + bdsvc = svc.NewCouchbase(couchbase.DBSettings{ + ConnectString: cfg.LocalIndexDirectory.Couchbase.ConnectString, + Auth: couchbase.DBSettingsAuth{ + Username: cfg.LocalIndexDirectory.Couchbase.Username, + Password: cfg.LocalIndexDirectory.Couchbase.Password, + }, + PieceMetadataBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: cfg.LocalIndexDirectory.Couchbase.PieceMetadataBucket.RAMQuotaMB, + }, + MultihashToPiecesBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: cfg.LocalIndexDirectory.Couchbase.MultihashToPiecesBucket.RAMQuotaMB, + }, + PieceOffsetsBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: cfg.LocalIndexDirectory.Couchbase.PieceOffsetsBucket.RAMQuotaMB, + }, + }) + } else { + log.Infow("local index directory: connecting to leveldb instance") + + // Setup a local index directory service that connects to the leveldb + var err error + bdsvc, err = svc.NewLevelDB(r.Path()) + if err != nil { + return fmt.Errorf("creating leveldb local index directory: %w", err) + } + } + + // Start the embedded local index directory service + err := bdsvc.Start(svcCtx, port) + if err != nil { + return fmt.Errorf("starting local index directory service: %w", err) + } + + // Connect to the embedded service + return client.Dial(ctx, fmt.Sprintf("http://localhost:%d", port)) + }, + OnStop: func(ctx context.Context) error { + cancel() + client.Close(ctx) + return nil + }, + }) + + return client + } +} + +func NewPieceDirectory(cfg *config.Boost) func(lc fx.Lifecycle, maddr dtypes.MinerAddress, store types.Store, secb sectorblocks.SectorBuilder, pp sealer.PieceProvider, full v1api.FullNode) *piecedirectory.PieceDirectory { + return func(lc fx.Lifecycle, maddr dtypes.MinerAddress, store types.Store, secb sectorblocks.SectorBuilder, pp sealer.PieceProvider, full v1api.FullNode) *piecedirectory.PieceDirectory { + sa := sectoraccessor.NewSectorAccessor(maddr, secb, pp, full) + pr := &piecedirectory.SectorAccessorAsPieceReader{SectorAccessor: sa} + pd := piecedirectory.NewPieceDirectory(store, pr, cfg.LocalIndexDirectory.ParallelAddIndexLimit) + + pdctx, cancel := context.WithCancel(context.Background()) + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + pd.Start(pdctx) + return nil + }, + OnStop: func(ctx context.Context) error { + cancel() + return nil + }, + }) + + return pd + } +} + +func NewPieceStore(pm *piecedirectory.PieceDirectory, maddr address.Address) piecestore.PieceStore { + return &boostPieceStoreWrapper{piecedirectory: pm, maddr: maddr} +} + +func NewPieceDoctor(lc fx.Lifecycle, store types.Store, sapi mktsdagstore.SectorAccessor) *piecedirectory.Doctor { + doc := piecedirectory.NewDoctor(store, sapi) + docctx, cancel := context.WithCancel(context.Background()) + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + go doc.Run(docctx) + return nil + }, + OnStop: func(ctx context.Context) error { + cancel() + return nil + }, + }) + return doc +} + +type boostPieceStoreWrapper struct { + piecedirectory *piecedirectory.PieceDirectory + maddr address.Address +} + +func (pw *boostPieceStoreWrapper) Start(ctx context.Context) error { + return nil +} + +func (pw *boostPieceStoreWrapper) OnReady(ready shared.ReadyFunc) { + go ready(nil) +} + +func (pw *boostPieceStoreWrapper) AddDealForPiece(pieceCID cid.Cid, proposalCid cid.Cid, dealInfo piecestore.DealInfo) error { + di := model.DealInfo{ + DealUuid: proposalCid.String(), + IsLegacy: true, + ChainDealID: dealInfo.DealID, + MinerAddr: pw.maddr, + SectorID: dealInfo.SectorID, + PieceOffset: dealInfo.Offset, + PieceLength: dealInfo.Length, + // TODO: It would be nice if there's some way to figure out the CAR + // file size here (but I don't think there is an easy way in legacy + // markets without having access to the piece data itself) + CarLength: 0, + } + return pw.piecedirectory.AddDealForPiece(context.Background(), pieceCID, di) +} + +func (pw *boostPieceStoreWrapper) AddPieceBlockLocations(pieceCID cid.Cid, blockLocations map[cid.Cid]piecestore.BlockLocation) error { + // This method is no longer needed, we keep the CAR file index in the piece metadata store + return nil +} + +func (pw *boostPieceStoreWrapper) GetPieceInfo(pieceCID cid.Cid) (piecestore.PieceInfo, error) { + pieceDeals, err := pw.piecedirectory.GetPieceDeals(context.TODO(), pieceCID) + if err != nil { + return piecestore.PieceInfo{}, fmt.Errorf("getting piece deals from piece metadata store: %w", err) + } + + dis := make([]piecestore.DealInfo, 0, len(pieceDeals)) + for _, pd := range pieceDeals { + dis = append(dis, piecestore.DealInfo{ + DealID: pd.ChainDealID, + SectorID: pd.SectorID, + Offset: pd.PieceOffset, + Length: pd.PieceLength, + }) + } + return piecestore.PieceInfo{ + PieceCID: pieceCID, + Deals: dis, + }, nil +} + +func (pw *boostPieceStoreWrapper) GetCIDInfo(payloadCID cid.Cid) (piecestore.CIDInfo, error) { + // This is no longer used (CLI calls piece metadata store instead) + return piecestore.CIDInfo{}, nil +} + +func (pw *boostPieceStoreWrapper) ListCidInfoKeys() ([]cid.Cid, error) { + // This is no longer used (CLI calls piece metadata store instead) + return nil, nil +} + +func (pw *boostPieceStoreWrapper) ListPieceInfoKeys() ([]cid.Cid, error) { + // This is no longer used (CLI calls piece metadata store instead) + return nil, nil +} + +func NewDAGStoreWrapper(pm *piecedirectory.PieceDirectory) stores.DAGStoreWrapper { + // TODO: lotus_modules.NewStorageMarketProvider and lotus_modules.RetrievalProvider + // take a concrete *dagstore.Wrapper as a parameter. Create boost versions of these + // that instead take a stores.DAGStoreWrapper parameter + return &boostDAGStoreWrapper{piecedirectory: pm} +} + +type boostDAGStoreWrapper struct { + piecedirectory *piecedirectory.PieceDirectory +} + +func (dw *boostDAGStoreWrapper) DestroyShard(ctx context.Context, pieceCid cid.Cid, resch chan dagstore.ShardResult) error { + // This is no longer used (CLI calls piece metadata store instead) + return nil +} + +// Legacy markets calls piecestore.AddDealForPiece before RegisterShard, +// so we do the real work in AddDealForPiece. +func (dw *boostDAGStoreWrapper) RegisterShard(ctx context.Context, pieceCid cid.Cid, carPath string, eagerInit bool, resch chan dagstore.ShardResult) error { + res := dagstore.ShardResult{ + Key: shard.KeyFromCID(pieceCid), + Error: nil, + Accessor: nil, + } + + select { + case resch <- res: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (dw *boostDAGStoreWrapper) LoadShard(ctx context.Context, pieceCid cid.Cid) (stores.ClosableBlockstore, error) { + bs, err := dw.piecedirectory.GetBlockstore(ctx, pieceCid) + if err != nil { + return nil, fmt.Errorf("getting blockstore in LoadShard: %w", err) + } + return closableBlockstore{Blockstore: bs}, nil +} + +func (dw *boostDAGStoreWrapper) MigrateDeals(ctx context.Context, deals []storagemarket.MinerDeal) (bool, error) { + // MigrateDeals is no longer needed - it's handled by the piece metadata store + return false, nil +} + +func (dw *boostDAGStoreWrapper) GetPiecesContainingBlock(blockCID cid.Cid) ([]cid.Cid, error) { + return dw.piecedirectory.PiecesContainingMultihash(context.TODO(), blockCID.Hash()) +} + +func (dw *boostDAGStoreWrapper) GetIterableIndexForPiece(pieceCid cid.Cid) (carindex.IterableIndex, error) { + return dw.piecedirectory.GetIterableIndex(context.TODO(), pieceCid) +} + +func (dw *boostDAGStoreWrapper) Close() error { + return nil +} + +type closableBlockstore struct { + bstore.Blockstore +} + +func (c closableBlockstore) Close() error { + return nil +} diff --git a/node/modules/provider_piece_store.go b/node/modules/provider_piece_store.go new file mode 100644 index 000000000..24ac7faeb --- /dev/null +++ b/node/modules/provider_piece_store.go @@ -0,0 +1,24 @@ +package modules + +import ( + "context" + + marketevents "github.com/filecoin-project/boost/markets/loggers" + "github.com/filecoin-project/boost/piecedirectory" + "github.com/filecoin-project/go-address" + lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" + "go.uber.org/fx" +) + +// NewProviderPieceStore creates a statestore for storing metadata about pieces +// shared by the storage and retrieval providers +func NewProviderPieceStore(lc fx.Lifecycle, pm *piecedirectory.PieceDirectory, maddr lotus_dtypes.MinerAddress) (lotus_dtypes.ProviderPieceStore, error) { + ps := NewPieceStore(pm, address.Address(maddr)) + ps.OnReady(marketevents.ReadyLogger("piecestore")) + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + return ps.Start(ctx) + }, + }) + return ps, nil +} diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 4033457f2..2b98eedd1 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/boost/node/impl/backupmgr" "github.com/filecoin-project/boost/node/modules/dtypes" + "github.com/filecoin-project/boost/piecedirectory" brm "github.com/filecoin-project/boost/retrievalmarket/lib" "github.com/filecoin-project/boost/retrievalmarket/rtvllog" "github.com/filecoin-project/boost/storagemanager" @@ -50,7 +51,6 @@ import ( "github.com/filecoin-project/lotus/gateway" "github.com/filecoin-project/lotus/journal" "github.com/filecoin-project/lotus/lib/sigs" - mdagstore "github.com/filecoin-project/lotus/markets/dagstore" "github.com/filecoin-project/lotus/node/modules" lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/node/modules/helpers" @@ -538,12 +538,12 @@ func NewLegacyStorageProvider(cfg *config.Boost) func(minerAddress lotus_dtypes. } } -func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, a v1api.FullNode, sqldb *sql.DB, dealsDB *db.DealsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, dp *storageadapter.DealPublisher, secb *sectorblocks.SectorBlocks, commpc types.CommpCalculator, sps sealingpipeline.API, df dtypes.StorageDealFilter, logsSqlDB *LogSqlDB, logsDB *db.LogsDB, dagst *mdagstore.Wrapper, ps lotus_dtypes.ProviderPieceStore, ip *indexprovider.Wrapper, lp lotus_storagemarket.StorageProvider, cdm *storagemarket.ChainDealManager) (*storagemarket.Provider, error) { +func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func(lc fx.Lifecycle, h host.Host, a v1api.FullNode, sqldb *sql.DB, dealsDB *db.DealsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, dp *storageadapter.DealPublisher, secb *sectorblocks.SectorBlocks, commpc types.CommpCalculator, sps sealingpipeline.API, df dtypes.StorageDealFilter, logsSqlDB *LogSqlDB, logsDB *db.LogsDB, piecedirectory *piecedirectory.PieceDirectory, ip *indexprovider.Wrapper, lp lotus_storagemarket.StorageProvider, cdm *storagemarket.ChainDealManager) (*storagemarket.Provider, error) { return func(lc fx.Lifecycle, h host.Host, a v1api.FullNode, sqldb *sql.DB, dealsDB *db.DealsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, dp *storageadapter.DealPublisher, secb *sectorblocks.SectorBlocks, commpc types.CommpCalculator, sps sealingpipeline.API, df dtypes.StorageDealFilter, logsSqlDB *LogSqlDB, logsDB *db.LogsDB, - dagst *mdagstore.Wrapper, ps lotus_dtypes.ProviderPieceStore, ip *indexprovider.Wrapper, + piecedirectory *piecedirectory.PieceDirectory, ip *indexprovider.Wrapper, lp lotus_storagemarket.StorageProvider, cdm *storagemarket.ChainDealManager) (*storagemarket.Provider, error) { prvCfg := storagemarket.Config{ @@ -562,7 +562,7 @@ func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func( dl := logs.NewDealLogger(logsDB) tspt := httptransport.New(h, dl) prov, err := storagemarket.NewProvider(prvCfg, sqldb, dealsDB, fundMgr, storageMgr, a, dp, provAddr, secb, commpc, - sps, cdm, df, logsSqlDB.db, logsDB, dagst, ps, ip, lp, &signatureVerifier{a}, dl, tspt) + sps, cdm, df, logsSqlDB.db, logsDB, piecedirectory, ip, lp, &signatureVerifier{a}, dl, tspt) if err != nil { return nil, err } @@ -571,18 +571,22 @@ func NewStorageMarketProvider(provAddr address.Address, cfg *config.Boost) func( } } -func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, ps lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, dagst dagstore.Interface, fullNode v1api.FullNode) *gql.Server { +func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, ps lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode) *gql.Server { return func(lc fx.Lifecycle, r repo.LockedRepo, h host.Host, prov *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, retDB *rtvllog.RetrievalLogDB, plDB *db.ProposalLogsDB, fundsDB *db.FundsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, publisher *storageadapter.DealPublisher, spApi sealingpipeline.API, legacyProv lotus_storagemarket.StorageProvider, legacyDT lotus_dtypes.ProviderDataTransfer, - ps lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, dagst dagstore.Interface, fullNode v1api.FullNode) *gql.Server { + ps lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, piecedirectory *piecedirectory.PieceDirectory, fullNode v1api.FullNode) *gql.Server { - resolver := gql.NewResolver(cfg, r, h, dealsDB, logsDB, retDB, plDB, fundsDB, fundMgr, storageMgr, spApi, prov, legacyProv, legacyDT, ps, sa, dagst, publisher, fullNode) + resolverCtx, cancel := context.WithCancel(context.Background()) + resolver := gql.NewResolver(resolverCtx, cfg, r, h, dealsDB, logsDB, retDB, plDB, fundsDB, fundMgr, storageMgr, spApi, prov, legacyProv, legacyDT, ps, sa, piecedirectory, publisher, fullNode) server := gql.NewServer(resolver) lc.Append(fx.Hook{ OnStart: server.Start, - OnStop: server.Stop, + OnStop: func(ctx context.Context) error { + cancel() + return server.Stop(ctx) + }, }) return server diff --git a/piecedirectory/doctor.go b/piecedirectory/doctor.go new file mode 100644 index 000000000..805b5359f --- /dev/null +++ b/piecedirectory/doctor.go @@ -0,0 +1,149 @@ +package piecedirectory + +import ( + "context" + "errors" + "fmt" + "math/rand" + "time" + + "github.com/filecoin-project/boost/piecedirectory/types" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" +) + +var doclog = logging.Logger("piecedoc") + +type SealingApi interface { + IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) +} + +// The Doctor periodically queries the local index directory for piece cids, and runs +// checks against those pieces. If there is a problem with a piece, it is +// flagged, so that it can be surfaced to the user. +// Note that multiple Doctor processes can run in parallel. The logic for which +// pieces to give to the Doctor to check is in the local index directory. +type Doctor struct { + store types.Store + sapi SealingApi +} + +func NewDoctor(store types.Store, sapi SealingApi) *Doctor { + return &Doctor{store: store, sapi: sapi} +} + +// The average interval between calls to NextPiecesToCheck +const avgCheckInterval = 30 * time.Second + +func (d *Doctor) Run(ctx context.Context) { + timer := time.NewTimer(0) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + } + + // Get the next pieces to check (eg pieces that haven't been checked + // for a while) from the local index directory + pcids, err := d.store.NextPiecesToCheck(ctx) + if err != nil { + if errors.Is(err, context.Canceled) { + return + } + doclog.Errorw("getting next pieces to check", "err", err) + time.Sleep(time.Minute) + continue + } + + // Check each piece for problems + doclog.Debugw("piece doctor: checking pieces", "count", len(pcids)) + for _, pcid := range pcids { + err := d.checkPiece(ctx, pcid) + if err != nil { + if errors.Is(err, context.Canceled) { + return + } + doclog.Errorw("checking piece", "piece", pcid, "err", err) + } + } + doclog.Debugw("piece doctor: completed checking pieces", "count", len(pcids)) + + // Sleep for a few seconds between ticks. + // The time to sleep is randomized, so that if there are multiple doctor + // processes they will each process some pieces some of the time. + sleepTime := avgCheckInterval/2 + time.Duration(rand.Intn(int(avgCheckInterval)))*time.Millisecond + timer.Reset(sleepTime) + } +} + +func (d *Doctor) checkPiece(ctx context.Context, pieceCid cid.Cid) error { + md, err := d.store.GetPieceMetadata(ctx, pieceCid) + if err != nil { + return fmt.Errorf("failed to get piece %s from local index directory: %w", pieceCid, err) + } + + // Check if the piece is in an error state + if md.Error != "" { + err = d.store.FlagPiece(ctx, pieceCid) + if err != nil { + return fmt.Errorf("failed to flag piece in error state %s: %w", pieceCid, err) + } + doclog.Debugw("piece is in error state", "err", md.Error) + return nil + } + + // Check if piece has been indexed + isIndexed, err := d.store.IsIndexed(ctx, pieceCid) + if err != nil { + return fmt.Errorf("failed to check index status of piece %s: %w", pieceCid, err) + } + + if !isIndexed { + err = d.store.FlagPiece(ctx, pieceCid) + if err != nil { + return fmt.Errorf("failed to flag unindexed piece %s: %w", pieceCid, err) + } + doclog.Debugw("flagging piece as unindexed", "piece", pieceCid) + return nil + } + + // Check if there is an unsealed copy of the piece + var hasUnsealedDeal bool + dls := md.Deals + for _, dl := range dls { + isUnsealedCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + isUnsealed, err := d.sapi.IsUnsealed(isUnsealedCtx, dl.SectorID, dl.PieceOffset.Unpadded(), dl.PieceLength.Unpadded()) + cancel() + if err != nil { + return fmt.Errorf("failed to check unsealed status of piece %s (sector %d, offset %d, length %d): %w", + pieceCid, dl.SectorID, dl.PieceOffset.Unpadded(), dl.PieceLength.Unpadded(), err) + } + + if isUnsealed { + hasUnsealedDeal = true + break + } + } + + if !hasUnsealedDeal { + err = d.store.FlagPiece(ctx, pieceCid) + if err != nil { + return fmt.Errorf("failed to flag piece %s with no unsealed deal: %w", pieceCid, err) + } + + doclog.Debugw("flagging piece as having no unsealed copy", "piece", pieceCid) + return nil + } + + // There are no known issues with the piece, so unflag it + err = d.store.UnflagPiece(ctx, pieceCid) + if err != nil { + return fmt.Errorf("failed to unflag piece %s: %w", pieceCid, err) + } + + return nil +} diff --git a/piecedirectory/doctor_test.go b/piecedirectory/doctor_test.go new file mode 100644 index 000000000..353266e15 --- /dev/null +++ b/piecedirectory/doctor_test.go @@ -0,0 +1,329 @@ +package piecedirectory + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/filecoin-project/boostd-data/client" + "github.com/filecoin-project/boostd-data/couchbase" + "github.com/filecoin-project/boostd-data/ldb" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/go-state-types/abi" + "github.com/google/uuid" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipld/go-car/v2" + "github.com/stretchr/testify/require" +) + +func TestPieceDoctor(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + + t.Run("leveldb", func(t *testing.T) { + prev := ldb.MinPieceCheckPeriod + ldb.MinPieceCheckPeriod = 1 * time.Second + + prevp := ldb.PiecesToTrackerBatchSize + ldb.PiecesToTrackerBatchSize = 4 + + bdsvc, err := svc.NewLevelDB("") + require.NoError(t, err) + + port := 8050 + err = bdsvc.Start(ctx, port) + require.NoError(t, err) + + cl := client.NewStore() + err = cl.Dial(ctx, fmt.Sprintf("http://localhost:%d", port)) + require.NoError(t, err) + defer cl.Close(ctx) + + t.Run("next pieces pagination", func(t *testing.T) { + testNextPiecesPagination(ctx, t, cl, ldb.MinPieceCheckPeriod) + }) + + t.Run("check pieces", func(t *testing.T) { + testCheckPieces(ctx, t, cl) + }) + + ldb.MinPieceCheckPeriod = prev + ldb.PiecesToTrackerBatchSize = prevp + }) + t.Run("couchbase", func(t *testing.T) { + // TODO: Unskip this test once the couchbase instance can be created + // from a docker container in CI + t.Skip() + + prev := couchbase.MinPieceCheckPeriod + couchbase.MinPieceCheckPeriod = 1 * time.Second + + svc.SetupCouchbase(t, testCouchSettings) + bdsvc := svc.NewCouchbase(testCouchSettings) + + port := 8051 + err := bdsvc.Start(ctx, port) + require.NoError(t, err) + + cl := client.NewStore() + err = cl.Dial(ctx, fmt.Sprintf("http://localhost:%d", port)) + require.NoError(t, err) + defer cl.Close(ctx) + + t.Run("next pieces", func(t *testing.T) { + testNextPieces(ctx, t, cl, couchbase.MinPieceCheckPeriod) + }) + + t.Run("check pieces", func(t *testing.T) { + testCheckPieces(ctx, t, cl) + }) + + couchbase.MinPieceCheckPeriod = prev + }) +} + +// Verify that after a new piece is added +// - NextPiecesToCheck immediately returns the piece +// - NextPiecesToCheck returns the piece every pieceCheckPeriod +func testNextPieces(ctx context.Context, t *testing.T, cl *client.Store, pieceCheckPeriod time.Duration) { + // Add a new piece + pieceCid := blocks.NewBlock([]byte(fmt.Sprintf("%d", time.Now().UnixMilli()))).Cid() + fmt.Println(pieceCid) + di := model.DealInfo{ + DealUuid: uuid.New().String(), + ChainDealID: 1, + SectorID: 1, + PieceOffset: 0, + PieceLength: 2048, + } + err := cl.AddDealForPiece(ctx, pieceCid, di) + require.NoError(t, err) + + // Sleep for half the piece check period + time.Sleep(pieceCheckPeriod / 2) + + // NextPiecesToCheck should return the piece (because it hasn't been checked yet) + pcids, err := cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Contains(t, pcids, pieceCid) + + // Calling NextPiecesToCheck again should return nothing, because the piece + // was just checked + pcids, err = cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.NotContains(t, pcids, pieceCid) + + // Sleep for at least the piece check period + time.Sleep(2 * pieceCheckPeriod) + + // Calling NextPiecesToCheck should return the piece, because it has not + // been checked for at least one piece check period + pcids, err = cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Contains(t, pcids, pieceCid) +} + +func testNextPiecesPagination(ctx context.Context, t *testing.T, cl *client.Store, pieceCheckPeriod time.Duration) { + // Add 8 pieces + allPcids := make(map[cid.Cid]struct{}) + seen := make(map[cid.Cid]int) + + for i := 1; i <= 9; i++ { + pieceCid := blocks.NewBlock([]byte(fmt.Sprintf("%d%d", time.Now().UnixMilli(), i))).Cid() + fmt.Println(pieceCid) + di := model.DealInfo{ + DealUuid: uuid.New().String(), + ChainDealID: abi.DealID(i), + SectorID: abi.SectorNumber(i), + PieceOffset: 0, + PieceLength: 2048, + } + err := cl.AddDealForPiece(ctx, pieceCid, di) + require.NoError(t, err) + + allPcids[pieceCid] = struct{}{} + } + + // expect to get 4 pieces + pcids, err := cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Len(t, pcids, 4) + for _, cid := range pcids { + seen[cid] = 1 + } + + // expect to get 4 pieces + pcids, err = cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Len(t, pcids, 4) + for _, cid := range pcids { + seen[cid] = 1 + } + + // expect to get 1 pieces (end of table) + pcids, err = cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Len(t, pcids, 1) + for _, cid := range pcids { + seen[cid] = 1 + } + + require.Len(t, seen, 9) + + // expect to get 0 pieces (first four) + pcids, err = cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Len(t, pcids, 0) + + // expect to get 0 pieces (second four) + pcids, err = cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Len(t, pcids, 0) + + // expect to get 0 pieces (end of table) + pcids, err = cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Len(t, pcids, 0) + + // Add 1 more piece + for i := 1; i <= 1; i++ { + pieceCid := blocks.NewBlock([]byte(fmt.Sprintf("%d%d", time.Now().UnixMilli(), i))).Cid() + fmt.Println(pieceCid) + di := model.DealInfo{ + DealUuid: uuid.New().String(), + ChainDealID: abi.DealID(i), + SectorID: abi.SectorNumber(i), + PieceOffset: 0, + PieceLength: 2048, + } + err := cl.AddDealForPiece(ctx, pieceCid, di) + require.NoError(t, err) + + allPcids[pieceCid] = struct{}{} + } + + // wait to reset the interval and start from scratch + time.Sleep(2 * time.Second) + seen = make(map[cid.Cid]int) + + pcids, err = cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Len(t, pcids, 4) + for _, cid := range pcids { + seen[cid] = 1 + } + + pcids, err = cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Len(t, pcids, 4) + for _, cid := range pcids { + seen[cid] = 1 + } + + pcids, err = cl.NextPiecesToCheck(ctx) + require.NoError(t, err) + require.Len(t, pcids, 2) + for _, cid := range pcids { + seen[cid] = 1 + } + + require.Len(t, seen, 10) +} + +func testCheckPieces(ctx context.Context, t *testing.T, cl *client.Store) { + // Create a random CAR file + carFilePath := CreateCarFile(t) + carFile, err := os.Open(carFilePath) + require.NoError(t, err) + defer carFile.Close() + + carReader, err := car.OpenReader(carFilePath) + require.NoError(t, err) + defer carReader.Close() + carv1Reader, err := carReader.DataReader() + require.NoError(t, err) + + commpCalc := CalculateCommp(t, carv1Reader) + + // Add deal info for the piece + di := model.DealInfo{ + DealUuid: uuid.New().String(), + ChainDealID: 1, + SectorID: 1, + PieceOffset: 0, + PieceLength: commpCalc.PieceSize, + } + err = cl.AddDealForPiece(ctx, commpCalc.PieceCID, di) + require.NoError(t, err) + + // Create a doctor + sapi := CreateMockDoctorSealingApi() + doc := NewDoctor(cl, sapi) + + // Check the piece + err = doc.checkPiece(ctx, commpCalc.PieceCID) + require.NoError(t, err) + + // The piece should be flagged because there is no index for it + count, err := cl.FlaggedPiecesCount(ctx) + require.NoError(t, err) + require.Equal(t, 1, count) + + pcids, err := cl.FlaggedPiecesList(ctx, nil, 0, 10) + require.NoError(t, err) + require.Equal(t, 1, len(pcids)) + + // Create an index for the piece + recs := GetRecords(t, carv1Reader) + err = cl.AddIndex(ctx, commpCalc.PieceCID, recs, true) + require.NoError(t, err) + + // Check the piece + err = doc.checkPiece(ctx, commpCalc.PieceCID) + require.NoError(t, err) + + // The piece should no longer be flagged + count, err = cl.FlaggedPiecesCount(ctx) + require.NoError(t, err) + require.Equal(t, 0, count) + + pcids, err = cl.FlaggedPiecesList(ctx, nil, 0, 10) + require.NoError(t, err) + require.Equal(t, 0, len(pcids)) + + // Mark the piece as not being unsealed + sapi.isUnsealed = false + + // Check the piece + err = doc.checkPiece(ctx, commpCalc.PieceCID) + require.NoError(t, err) + + // The piece should be flagged because there is no unsealed copy + count, err = cl.FlaggedPiecesCount(ctx) + require.NoError(t, err) + require.Equal(t, 1, count) + + pcids, err = cl.FlaggedPiecesList(ctx, nil, 0, 10) + require.NoError(t, err) + require.Equal(t, 1, len(pcids)) + + // Mark the piece as being unsealed + sapi.isUnsealed = true + + // Check the piece + err = doc.checkPiece(ctx, commpCalc.PieceCID) + require.NoError(t, err) + + // The piece should no longer be flagged + count, err = cl.FlaggedPiecesCount(ctx) + require.NoError(t, err) + require.Equal(t, 0, count) + + pcids, err = cl.FlaggedPiecesList(ctx, nil, 0, 10) + require.NoError(t, err) + require.Equal(t, 0, len(pcids)) +} diff --git a/piecedirectory/piecedirectory.go b/piecedirectory/piecedirectory.go new file mode 100644 index 000000000..5b3aecef6 --- /dev/null +++ b/piecedirectory/piecedirectory.go @@ -0,0 +1,636 @@ +package piecedirectory + +import ( + "bufio" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "sync" + "time" + + "github.com/filecoin-project/boost/piecedirectory/types" + "github.com/filecoin-project/boostd-data/client" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/shared/tracing" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/markets/dagstore" + "github.com/hashicorp/go-multierror" + "github.com/ipfs/go-cid" + bstore "github.com/ipfs/go-ipfs-blockstore" + format "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log/v2" + "github.com/ipld/go-car/util" + carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/blockstore" + carindex "github.com/ipld/go-car/v2/index" + "github.com/multiformats/go-multihash" + mh "github.com/multiformats/go-multihash" +) + +var log = logging.Logger("piecedirectory") + +type PieceDirectory struct { + store types.Store + pieceReader types.PieceReader + + ctx context.Context + + addIdxThrottleSize int + addIdxThrottle chan struct{} + addIdxOpByCid sync.Map +} + +func NewStore() *client.Store { + return client.NewStore() +} + +func NewPieceDirectory(store types.Store, pr types.PieceReader, addIndexThrottleSize int) *PieceDirectory { + return &PieceDirectory{ + store: store, + pieceReader: pr, + addIdxThrottleSize: addIndexThrottleSize, + addIdxThrottle: make(chan struct{}, addIndexThrottleSize), + } +} + +func (ps *PieceDirectory) Start(ctx context.Context) { + ps.ctx = ctx +} + +func (ps *PieceDirectory) FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) { + return ps.store.FlaggedPiecesList(ctx, cursor, offset, limit) +} + +func (ps *PieceDirectory) FlaggedPiecesCount(ctx context.Context) (int, error) { + return ps.store.FlaggedPiecesCount(ctx) +} + +// Get all metadata about a particular piece +func (ps *PieceDirectory) GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (types.PieceDirMetadata, error) { + ctx, span := tracing.Tracer.Start(ctx, "pm.get_piece_metadata") + defer span.End() + + // Get the piece metadata from the DB + log.Debugw("piece metadata: get", "pieceCid", pieceCid) + md, err := ps.store.GetPieceMetadata(ctx, pieceCid) + if err != nil { + return types.PieceDirMetadata{}, err + } + + // Check if this process is currently indexing the piece + log.Debugw("piece metadata: get indexing status", "pieceCid", pieceCid) + _, indexing := ps.addIdxOpByCid.Load(pieceCid) + + // Return the db piece metadata along with the indexing flag + log.Debugw("piece metadata: get complete", "pieceCid", pieceCid) + return types.PieceDirMetadata{ + Metadata: md, + Indexing: indexing, + }, nil +} + +// Get the list of deals (and the sector the data is in) for a particular piece +func (ps *PieceDirectory) GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) { + ctx, span := tracing.Tracer.Start(ctx, "pm.get_piece_deals") + defer span.End() + + deals, err := ps.store.GetPieceDeals(ctx, pieceCid) + if err != nil { + return nil, fmt.Errorf("listing deals for piece %s: %w", pieceCid, err) + } + + return deals, nil +} + +func (ps *PieceDirectory) GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash mh.Multihash) (*model.OffsetSize, error) { + ctx, span := tracing.Tracer.Start(ctx, "pm.get_offset") + defer span.End() + + return ps.store.GetOffsetSize(ctx, pieceCid, hash) +} + +func (ps *PieceDirectory) GetCarSize(ctx context.Context, pieceCid cid.Cid) (uint64, error) { + // Get the deals for the piece + dls, err := ps.GetPieceDeals(ctx, pieceCid) + if err != nil { + return 0, fmt.Errorf("getting piece deals for piece %s: %w", pieceCid, err) + } + + if len(dls) == 0 { + return 0, fmt.Errorf("no deals for piece %s in index: piece not found", pieceCid) + } + + // The size of the CAR should be the same for any deal, so just return the + // first non-zero CAR size + for _, dl := range dls { + if dl.CarLength > 0 { + return dl.CarLength, nil + } + } + + // There are no deals with a non-zero CAR size. + // The CAR size is zero if it's been imported from the dagstore (the + // dagstore doesn't store CAR size information). So instead work out the + // size of the CAR by getting the offset of the last section in the CAR + // file, then reading the section information. + + // Get the offset of the last section in the CAR file from the index. + var lastSectionOffset uint64 + idx, err := ps.GetIterableIndex(ctx, pieceCid) + if err != nil { + return 0, fmt.Errorf("getting index for piece %s: %w", pieceCid, err) + } + err = idx.ForEach(func(_ mh.Multihash, offset uint64) error { + if offset > lastSectionOffset { + lastSectionOffset = offset + } + return nil + }) + if err != nil { + return 0, fmt.Errorf("iterating index for piece %s: %w", pieceCid, err) + } + + // Get a reader over the piece + pieceReader, err := ps.GetPieceReader(ctx, pieceCid) + if err != nil { + return 0, fmt.Errorf("getting piece reader for piece %s: %w", pieceCid, err) + } + + // Seek to the last section + _, err = pieceReader.Seek(int64(lastSectionOffset), io.SeekStart) + if err != nil { + return 0, fmt.Errorf("seeking to offset %d in piece data: %w", lastSectionOffset, err) + } + + // A section consists of + // + + // Get + cr := &countReader{r: bufio.NewReader(pieceReader)} + dataLength, err := binary.ReadUvarint(cr) + if err != nil { + return 0, fmt.Errorf("reading CAR section length: %w", err) + } + + // The number of bytes in the uvarint that records + dataLengthUvarSize := cr.count + + // Get the size of the (unpadded) CAR file + unpaddedCarSize := lastSectionOffset + dataLengthUvarSize + dataLength + + // Write the CAR size back to the store so that it's cached for next time + err = ps.store.SetCarSize(ctx, pieceCid, unpaddedCarSize) + if err != nil { + log.Errorw("writing CAR size to local index directory store", "pieceCid", pieceCid, "err", err) + } + + return unpaddedCarSize, nil +} + +func (ps *PieceDirectory) AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { + ctx, span := tracing.Tracer.Start(ctx, "pm.add_deal_for_piece") + defer span.End() + + // Check if the indexes have already been added + isIndexed, err := ps.store.IsIndexed(ctx, pieceCid) + if err != nil { + return err + } + + if !isIndexed { + // Perform indexing of piece + if err := ps.addIndexForPieceThrottled(ctx, pieceCid, dealInfo); err != nil { + return fmt.Errorf("adding index for piece %s: %w", pieceCid, err) + } + } + + // Add deal to list of deals for this piece + if err := ps.store.AddDealForPiece(ctx, pieceCid, dealInfo); err != nil { + return fmt.Errorf("saving deal %s to store: %w", dealInfo.DealUuid, err) + } + + return nil +} + +type addIndexOperation struct { + done chan struct{} + err error +} + +func (ps *PieceDirectory) addIndexForPieceThrottled(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { + // Check if there is already an add index operation in progress for the + // given piece cid. If not, create a new one. + opi, loaded := ps.addIdxOpByCid.LoadOrStore(pieceCid, &addIndexOperation{ + done: make(chan struct{}), + }) + op := opi.(*addIndexOperation) + if loaded { + log.Debugw("add index: operation in progress, waiting for completion", "pieceCid", pieceCid) + defer func() { + log.Debugw("add index: in progress operation completed", "pieceCid", pieceCid) + }() + + // There is an add index operation in progress, so wait for it to + // complete + select { + case <-ctx.Done(): + return ctx.Err() + case <-op.done: + return op.err + } + } + + // A new operation was added to the map, so clean it up when it's done + defer ps.addIdxOpByCid.Delete(pieceCid) + defer close(op.done) + + // Wait for the throttle to yield an open spot + log.Debugw("add index: wait for open throttle position", + "pieceCid", pieceCid, "queued", len(ps.addIdxThrottle), "queue-limit", ps.addIdxThrottleSize) + select { + case <-ctx.Done(): + op.err = ctx.Err() + return ctx.Err() + case ps.addIdxThrottle <- struct{}{}: + } + defer func() { <-ps.addIdxThrottle }() + + // Perform the add index operation. + // Note: Once we start the add index operation we don't want to cancel it + // if one of the waiting threads cancels its context. So instead we use the + // PieceDirectory's context. + op.err = ps.addIndexForPiece(ps.ctx, pieceCid, dealInfo) + + // Return the result + log.Debugw("add index: completed", "pieceCid", pieceCid) + return op.err +} + +func (ps *PieceDirectory) addIndexForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error { + // Get a reader over the piece data + log.Debugw("add index: get index", "pieceCid", pieceCid) + reader, err := ps.pieceReader.GetReader(ctx, dealInfo.SectorID, dealInfo.PieceOffset, dealInfo.PieceLength) + if err != nil { + return fmt.Errorf("getting reader over piece %s: %w", pieceCid, err) + } + + // Iterate over all the blocks in the piece to extract the index records + log.Debugw("add index: read index", "pieceCid", pieceCid) + recs := make([]model.Record, 0) + opts := []carv2.Option{carv2.ZeroLengthSectionAsEOF(true)} + blockReader, err := carv2.NewBlockReader(reader, opts...) + if err != nil { + return fmt.Errorf("getting block reader over piece %s: %w", pieceCid, err) + } + + blockMetadata, err := blockReader.SkipNext() + for err == nil { + recs = append(recs, model.Record{ + Cid: blockMetadata.Cid, + OffsetSize: model.OffsetSize{ + Offset: blockMetadata.Offset, + Size: blockMetadata.Size, + }, + }) + + blockMetadata, err = blockReader.SkipNext() + } + if !errors.Is(err, io.EOF) { + return fmt.Errorf("generating index for piece %s: %w", pieceCid, err) + } + + // Add mh => piece index to store: "which piece contains the multihash?" + // Add mh => offset index to store: "what is the offset of the multihash within the piece?" + log.Debugw("add index: store index in local index directory", "pieceCid", pieceCid) + if err := ps.store.AddIndex(ctx, pieceCid, recs, true); err != nil { + return fmt.Errorf("adding CAR index for piece %s: %w", pieceCid, err) + } + + return nil +} + +func (ps *PieceDirectory) BuildIndexForPiece(ctx context.Context, pieceCid cid.Cid) error { + ctx, span := tracing.Tracer.Start(ctx, "pm.build_index_for_piece") + defer span.End() + + log.Debugw("build index: get piece deals", "pieceCid", pieceCid) + + dls, err := ps.GetPieceDeals(ctx, pieceCid) + if err != nil { + return fmt.Errorf("getting piece deals: %w", err) + } + + if len(dls) == 0 { + log.Debugw("build index: no deals found for piece", "pieceCid", pieceCid) + return fmt.Errorf("getting piece deals: no deals found for piece") + } + + err = ps.addIndexForPieceThrottled(ctx, pieceCid, dls[0]) + if err != nil { + return fmt.Errorf("adding index for piece deal %d: %w", dls[0].ChainDealID, err) + } + + return nil +} + +func (ps *PieceDirectory) RemoveDealForPiece(ctx context.Context, pieceCid cid.Cid, dealUuid string) error { + ctx, span := tracing.Tracer.Start(ctx, "pm.delete_deal_for_piece") + defer span.End() + + //Delete deal from list of deals for this piece + //It removes metadata and indexes if []deal is empty + err := ps.store.RemoveDealForPiece(ctx, pieceCid, dealUuid) + if err != nil { + return fmt.Errorf("deleting deal from piece metadata: %w", err) + } + return nil +} + +func (ps *PieceDirectory) MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error { + return ps.store.MarkIndexErrored(ctx, pieceCid, err) +} + +//func (ps *piecedirectory) deleteIndexForPiece(pieceCid cid.Cid) interface{} { +// TODO: Maybe mark for GC instead of deleting immediately + +// Delete mh => offset index from store +//err := ps.carIndex.Delete(pieceCid) +//if err != nil { +//err = fmt.Errorf("deleting CAR index for piece %s: %w", pieceCid, err) +//} + +//// Delete mh => piece index from store +//if mherr := ps.mhToPieceIndex.Delete(pieceCid); mherr != nil { +//err = multierror.Append(fmt.Errorf("deleting cid index for piece %s: %w", pieceCid, mherr)) +//} +//return err +//return nil +//} + +// Used internally, and also by HTTP retrieval +func (ps *PieceDirectory) GetPieceReader(ctx context.Context, pieceCid cid.Cid) (types.SectionReader, error) { + ctx, span := tracing.Tracer.Start(ctx, "pm.get_piece_reader") + defer span.End() + + // Get all deals containing this piece + deals, err := ps.GetPieceDeals(ctx, pieceCid) + if err != nil { + return nil, fmt.Errorf("getting piece deals: %w", err) + } + + if len(deals) == 0 { + return nil, fmt.Errorf("no deals found for piece cid %s: %w", pieceCid, err) + } + + // For each deal, try to read an unsealed copy of the data from the sector + // it is stored in + var merr error + for i, dl := range deals { + reader, err := ps.pieceReader.GetReader(ctx, dl.SectorID, dl.PieceOffset, dl.PieceLength) + if err != nil { + // TODO: log error + if i < 3 { + merr = multierror.Append(merr, err) + } + continue + } + + return reader, nil + } + + return nil, merr +} + +// Get all pieces that contain a multihash (used when retrieving by payload CID) +func (ps *PieceDirectory) PiecesContainingMultihash(ctx context.Context, m mh.Multihash) ([]cid.Cid, error) { + ctx, span := tracing.Tracer.Start(ctx, "pm.pieces_containing_multihash") + defer span.End() + + return ps.store.PiecesContainingMultihash(ctx, m) +} + +func (ps *PieceDirectory) GetIterableIndex(ctx context.Context, pieceCid cid.Cid) (carindex.IterableIndex, error) { + ctx, span := tracing.Tracer.Start(ctx, "pm.get_iterable_index") + defer span.End() + + idx, err := ps.store.GetIndex(ctx, pieceCid) + if err != nil { + return nil, err + } + + switch concrete := idx.(type) { + case carindex.IterableIndex: + return concrete, nil + default: + return nil, fmt.Errorf("expected index to be MultihashIndexSorted but got %T", idx) + } +} + +// Get a block (used by Bitswap retrieval) +func (ps *PieceDirectory) BlockstoreGet(ctx context.Context, c cid.Cid) ([]byte, error) { + // TODO: use caching to make this efficient for repeated Gets against the same piece + ctx, span := tracing.Tracer.Start(ctx, "pm.get_block") + defer span.End() + + // Get the pieces that contain the cid + pieces, err := ps.PiecesContainingMultihash(ctx, c.Hash()) + + // Check if it's an identity cid, if it is, return its digest + if err != nil { + digest, ok, err := isIdentity(c) + if err == nil && ok { + return digest, nil + } + return nil, fmt.Errorf("getting pieces containing cid %s: %w", c, err) + } + if len(pieces) == 0 { + return nil, fmt.Errorf("no pieces with cid %s found", c) + } + + // Get a reader over one of the pieces and extract the block data + var merr error + for i, pieceCid := range pieces { + data, err := func() ([]byte, error) { + // Get a reader over the piece data + reader, err := ps.GetPieceReader(ctx, pieceCid) + if err != nil { + return nil, fmt.Errorf("getting piece reader: %w", err) + } + defer reader.Close() + + // Get the offset of the block within the piece (CAR file) + offsetSize, err := ps.GetOffsetSize(ctx, pieceCid, c.Hash()) + if err != nil { + return nil, fmt.Errorf("getting offset/size for cid %s in piece %s: %w", c, pieceCid, err) + } + + // Seek to the block offset + _, err = reader.Seek(int64(offsetSize.Offset), io.SeekStart) + if err != nil { + return nil, fmt.Errorf("seeking to offset %d in piece reader: %w", int64(offsetSize.Offset), err) + } + + // Read the block data + _, data, err := util.ReadNode(bufio.NewReader(reader)) + if err != nil { + return nil, fmt.Errorf("reading data for block %s from reader for piece %s: %w", c, pieceCid, err) + } + return data, nil + }() + if err != nil { + if i < 3 { + merr = multierror.Append(merr, err) + } + continue + } + return data, nil + } + + return nil, merr +} + +func (ps *PieceDirectory) BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) { + ctx, span := tracing.Tracer.Start(ctx, "pm.get_block_size") + defer span.End() + + // Get the pieces that contain the cid + pieces, err := ps.PiecesContainingMultihash(ctx, c.Hash()) + if err != nil { + return 0, fmt.Errorf("getting pieces containing cid %s: %w", c, err) + } + if len(pieces) == 0 { + // We must return ipld ErrNotFound here because that's the only type + // that bitswap interprets as a not found error. All other error types + // are treated as general errors. + return 0, format.ErrNotFound{Cid: c} + } + + // Get the size of the block from the first piece (should be the same for + // any piece) + offsetSize, err := ps.GetOffsetSize(ctx, pieces[0], c.Hash()) + if err != nil { + return 0, fmt.Errorf("getting size of cid %s in piece %s: %w", c, pieces[0], err) + } + + if offsetSize.Size > 0 { + return int(offsetSize.Size), nil + } + + // Indexes imported from the DAG store do not have block size information + // (they only have offset information). Check if the block size is zero + // because the index is incomplete. + isComplete, err := ps.store.IsCompleteIndex(ctx, pieces[0]) + if err != nil { + return 0, fmt.Errorf("getting index complete status for piece %s: %w", pieces[0], err) + } + + if isComplete { + // The deal index is complete, so it must be a zero-sized block. + // A zero-sized block is unusual, but possible. + return int(offsetSize.Size), nil + } + + // The index is incomplete, so re-build the index on the fly + err = ps.BuildIndexForPiece(ctx, pieces[0]) + if err != nil { + return 0, fmt.Errorf("re-building index for piece %s: %w", pieces[0], err) + } + + // Now get the size again + offsetSize, err = ps.GetOffsetSize(ctx, pieces[0], c.Hash()) + if err != nil { + return 0, fmt.Errorf("getting size of cid %s in piece %s: %w", c, pieces[0], err) + } + + return int(offsetSize.Size), nil +} + +func (ps *PieceDirectory) BlockstoreHas(ctx context.Context, c cid.Cid) (bool, error) { + ctx, span := tracing.Tracer.Start(ctx, "pm.has_block") + defer span.End() + + // Get the pieces that contain the cid + pieces, err := ps.PiecesContainingMultihash(ctx, c.Hash()) + if err != nil { + return false, fmt.Errorf("getting pieces containing cid %s: %w", c, err) + } + return len(pieces) > 0, nil +} + +// Get a blockstore over a piece (used by Graphsync retrieval) +func (ps *PieceDirectory) GetBlockstore(ctx context.Context, pieceCid cid.Cid) (bstore.Blockstore, error) { + ctx, span := tracing.Tracer.Start(ctx, "pm.get_blockstore") + defer span.End() + + // Get a reader over the piece + reader, err := ps.GetPieceReader(ctx, pieceCid) + if err != nil { + return nil, fmt.Errorf("getting piece reader for piece %s: %w", pieceCid, err) + } + + // Get an index for the piece + idx, err := ps.GetIterableIndex(ctx, pieceCid) + if err != nil { + return nil, fmt.Errorf("getting index for piece %s: %w", pieceCid, err) + } + + // process index and store entries + // Create a blockstore from the index and the piece reader + bs, err := blockstore.NewReadOnly(reader, idx, carv2.ZeroLengthSectionAsEOF(true)) + if err != nil { + return nil, fmt.Errorf("creating blockstore for piece %s: %w", pieceCid, err) + } + + return bs, nil +} + +// countReader just counts the number of bytes read +type countReader struct { + r *bufio.Reader + count uint64 +} + +func (c *countReader) ReadByte() (byte, error) { + b, err := c.r.ReadByte() + if err == nil { + c.count++ + } + return b, err +} + +type SectorAccessorAsPieceReader struct { + dagstore.SectorAccessor +} + +func (s *SectorAccessorAsPieceReader) GetReader(ctx context.Context, id abi.SectorNumber, offset abi.PaddedPieceSize, length abi.PaddedPieceSize) (types.SectionReader, error) { + ctx, span := tracing.Tracer.Start(ctx, "sealer.get_reader") + defer span.End() + + isUnsealed, err := s.SectorAccessor.IsUnsealed(ctx, id, offset.Unpadded(), length.Unpadded()) + if err != nil { + return nil, fmt.Errorf("checking unsealed state of sector %d: %w", id, err) + } + + if !isUnsealed { + return nil, fmt.Errorf("getting reader over sector %d: %w", id, types.ErrSealed) + } + + r, err := s.SectorAccessor.UnsealSectorAt(ctx, id, offset.Unpadded(), length.Unpadded()) + if err != nil { + return nil, fmt.Errorf("getting reader over sector %d: %w", id, err) + } + + return r, nil +} + +func isIdentity(c cid.Cid) (digest []byte, ok bool, err error) { + dmh, err := multihash.Decode(c.Hash()) + if err != nil { + return nil, false, err + } + ok = dmh.Code == multihash.IDENTITY + digest = dmh.Digest + return digest, ok, nil +} diff --git a/piecedirectory/piecedirectory_test.go b/piecedirectory/piecedirectory_test.go new file mode 100644 index 000000000..08858330b --- /dev/null +++ b/piecedirectory/piecedirectory_test.go @@ -0,0 +1,365 @@ +package piecedirectory + +import ( + "bytes" + "context" + "io" + "os" + "testing" + "time" + + mock_piecedirectory "github.com/filecoin-project/boost/piecedirectory/types/mocks" + "github.com/filecoin-project/boostd-data/client" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/boostd-data/svc" + "github.com/filecoin-project/boostd-data/svc/types" + "github.com/golang/mock/gomock" + "github.com/google/uuid" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/blockstore" + "github.com/stretchr/testify/require" +) + +func TestPieceDirectory(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + + t.Run("leveldb", func(t *testing.T) { + bdsvc, err := svc.NewLevelDB("") + require.NoError(t, err) + testPieceDirectory(ctx, t, bdsvc) + }) + t.Run("couchbase", func(t *testing.T) { + // TODO: Unskip this test once the couchbase instance can be created + // from a docker container in CI as part of the test + t.Skip() + svc.SetupCouchbase(t, testCouchSettings) + bdsvc := svc.NewCouchbase(testCouchSettings) + testPieceDirectory(ctx, t, bdsvc) + }) +} + +func testPieceDirectory(ctx context.Context, t *testing.T, bdsvc *svc.Service) { + err := bdsvc.Start(ctx, 8044) + require.NoError(t, err) + + cl := client.NewStore() + err = cl.Dial(ctx, "http://localhost:8044") + require.NoError(t, err) + defer cl.Close(ctx) + + t.Run("not found", func(t *testing.T) { + testPieceDirectoryNotFound(ctx, t, cl) + }) + + t.Run("basic blockstore", func(t *testing.T) { + testBasicBlockstoreMethods(ctx, t, cl) + }) + + t.Run("imported index", func(t *testing.T) { + testImportedIndex(ctx, t, cl) + }) + + t.Run("car file size", func(t *testing.T) { + testCarFileSize(ctx, t, cl) + }) + + t.Run("flagging pieces", func(t *testing.T) { + testFlaggingPieces(ctx, t, cl) + }) +} + +func testPieceDirectoryNotFound(ctx context.Context, t *testing.T, cl *client.Store) { + ctrl := gomock.NewController(t) + pr := mock_piecedirectory.NewMockPieceReader(ctrl) + pm := NewPieceDirectory(cl, pr, 1) + pm.Start(ctx) + + nonExistentPieceCid, err := cid.Parse("bafkqaaa") + require.NoError(t, err) + + _, err = pm.GetPieceMetadata(ctx, nonExistentPieceCid) + require.True(t, types.IsNotFound(err)) + require.Error(t, err) + + _, err = pm.GetPieceDeals(ctx, nonExistentPieceCid) + require.True(t, types.IsNotFound(err)) + require.Error(t, err) + + _, err = pm.GetOffsetSize(ctx, nonExistentPieceCid, nonExistentPieceCid.Hash()) + require.True(t, types.IsNotFound(err)) + require.Error(t, err) + + _, err = pm.GetIterableIndex(ctx, nonExistentPieceCid) + require.True(t, types.IsNotFound(err)) + require.Error(t, err) + + _, err = pm.PiecesContainingMultihash(ctx, nonExistentPieceCid.Hash()) + require.True(t, types.IsNotFound(err)) + require.Error(t, err) +} + +// Verify that Has, GetSize and Get block work +func testBasicBlockstoreMethods(ctx context.Context, t *testing.T, cl *client.Store) { + carFilePath := CreateCarFile(t) + carFile, err := os.Open(carFilePath) + require.NoError(t, err) + defer carFile.Close() + + // Create a random CAR file + carReader, err := car.OpenReader(carFilePath) + require.NoError(t, err) + defer carReader.Close() + carv1Reader, err := carReader.DataReader() + require.NoError(t, err) + + // Any calls to get a reader over data should return a reader over the random CAR file + pr := CreateMockPieceReader(t, carv1Reader) + + pm := NewPieceDirectory(cl, pr, 1) + pm.Start(ctx) + pieceCid := CalculateCommp(t, carv1Reader).PieceCID + + // Add deal info for the piece - it doesn't matter what it is, the piece + // just needs to have at least one deal associated with it + di := model.DealInfo{ + DealUuid: uuid.New().String(), + ChainDealID: 1, + SectorID: 2, + PieceOffset: 0, + PieceLength: 0, + } + err = pm.AddDealForPiece(ctx, pieceCid, di) + require.NoError(t, err) + + // Create a CARv2 index over the CAR file + carFileIdxReader, err := os.Open(carFilePath) + require.NoError(t, err) + defer carFileIdxReader.Close() + idx, err := car.ReadOrGenerateIndex(carFileIdxReader) + require.NoError(t, err) + + // Create a blockstore from the CAR file and index + _, err = carFile.Seek(0, io.SeekStart) + require.NoError(t, err) + bs, err := blockstore.NewReadOnly(carFile, idx) + require.NoError(t, err) + + // Get the index (offset and size information) + recs := GetRecords(t, carv1Reader) + + // Verify that blockstore has, get and get size work + for _, rec := range recs { + blk, err := bs.Get(ctx, rec.Cid) + require.NoError(t, err) + + has, err := pm.BlockstoreHas(ctx, rec.Cid) + require.NoError(t, err) + require.True(t, has) + + sz, err := pm.BlockstoreGetSize(ctx, rec.Cid) + require.NoError(t, err) + require.Equal(t, len(blk.RawData()), sz) + + pmblk, err := pm.BlockstoreGet(ctx, rec.Cid) + require.NoError(t, err) + require.True(t, bytes.Equal(blk.RawData(), pmblk)) + } +} + +// Verify that if the index has been imported from the DAG store, meaning +// it has offset information but not block size information, the local index directory +// will re-build the index +func testImportedIndex(ctx context.Context, t *testing.T, cl *client.Store) { + // Create a random CAR file + carFilePath := CreateCarFile(t) + carFile, err := os.Open(carFilePath) + require.NoError(t, err) + defer carFile.Close() + + carReader, err := car.OpenReader(carFilePath) + require.NoError(t, err) + defer carReader.Close() + carv1Reader, err := carReader.DataReader() + require.NoError(t, err) + + // Any calls to get a reader over data should return a reader over the random CAR file + pr := CreateMockPieceReader(t, carv1Reader) + + recs := GetRecords(t, carv1Reader) + pieceCid := CalculateCommp(t, carv1Reader).PieceCID + err = cl.AddIndex(ctx, pieceCid, recs, false) + require.NoError(t, err) + + // Add deal info for the piece - it doesn't matter what it is, the piece + // just needs to have at least one deal associated with it + di := model.DealInfo{ + DealUuid: uuid.New().String(), + ChainDealID: 1, + SectorID: 2, + PieceOffset: 0, + PieceLength: 0, + } + err = cl.AddDealForPiece(ctx, pieceCid, di) + require.NoError(t, err) + + // Remove size information from the index records. + // This is to simulate what happens when an index is imported from the + // DAG store: only the offset information is imported (not size + // information) + for i, r := range recs { + recs[i] = model.Record{ + Cid: r.Cid, + OffsetSize: model.OffsetSize{ + Offset: r.Offset, + Size: 0, + }, + } + } + + // Add the index to the local index directory, marked as incomplete + err = cl.AddIndex(ctx, pieceCid, recs, false) + require.NoError(t, err) + + // Create a CARv2 index over the CAR file + carFileIdxReader, err := os.Open(carFilePath) + require.NoError(t, err) + defer carFileIdxReader.Close() + idx, err := car.ReadOrGenerateIndex(carFileIdxReader) + require.NoError(t, err) + + // Create a blockstore from the CAR file and index + _, err = carFile.Seek(0, io.SeekStart) + require.NoError(t, err) + bs, err := blockstore.NewReadOnly(carFile, idx) + require.NoError(t, err) + + // Pick a record in the middle of the DAG + rec := recs[len(recs)/2] + blk, err := bs.Get(ctx, rec.Cid) + require.NoError(t, err) + + // Verify that getting the size of a block works correctly: + // There is no size information in the index so the piece + // directory should re-build the index and then return the size. + pm := NewPieceDirectory(cl, pr, 1) + pm.Start(ctx) + sz, err := pm.BlockstoreGetSize(ctx, rec.Cid) + require.NoError(t, err) + require.Equal(t, len(blk.RawData()), sz) +} + +// Verify that if the deal info has been imported from the DAG store, meaning +// it does not have CAR size information, GetCarSize will correctly calculate +// the CAR size from the index + piece data +func testCarFileSize(ctx context.Context, t *testing.T, cl *client.Store) { + // Create a random CAR file + carFilePath := CreateCarFile(t) + carFile, err := os.Open(carFilePath) + require.NoError(t, err) + defer carFile.Close() + + carReader, err := car.OpenReader(carFilePath) + require.NoError(t, err) + defer carReader.Close() + carv1Reader, err := carReader.DataReader() + require.NoError(t, err) + + // Read the CAR bytes + carBytes, err := io.ReadAll(carv1Reader) + require.NoError(t, err) + + // Any calls to get a reader over data should return a reader over the random CAR file + pr := CreateMockPieceReader(t, carv1Reader) + + recs := GetRecords(t, carv1Reader) + commpCalc := CalculateCommp(t, carv1Reader) + err = cl.AddIndex(ctx, commpCalc.PieceCID, recs, false) + require.NoError(t, err) + + // Add deal info for the piece without a CAR file + di := model.DealInfo{ + DealUuid: uuid.New().String(), + ChainDealID: 1, + SectorID: 1, + PieceOffset: 0, + PieceLength: commpCalc.PieceSize, + } + err = cl.AddDealForPiece(ctx, commpCalc.PieceCID, di) + require.NoError(t, err) + + // Verify that getting the size of the CAR file works correctly: + // There is no CAR size information in the deal info, so the piece + // directory should work it out from the index and piece data. + pm := NewPieceDirectory(cl, pr, 1) + pm.Start(ctx) + size, err := pm.GetCarSize(ctx, commpCalc.PieceCID) + require.NoError(t, err) + require.Equal(t, len(carBytes), int(size)) +} + +func testFlaggingPieces(ctx context.Context, t *testing.T, cl *client.Store) { + // Create a random CAR file + carFilePath := CreateCarFile(t) + carFile, err := os.Open(carFilePath) + require.NoError(t, err) + defer carFile.Close() + + carReader, err := car.OpenReader(carFilePath) + require.NoError(t, err) + defer carReader.Close() + carv1Reader, err := carReader.DataReader() + require.NoError(t, err) + + recs := GetRecords(t, carv1Reader) + commpCalc := CalculateCommp(t, carv1Reader) + err = cl.AddIndex(ctx, commpCalc.PieceCID, recs, true) + require.NoError(t, err) + + // Add deal info for the piece + di := model.DealInfo{ + DealUuid: uuid.New().String(), + ChainDealID: 1, + SectorID: 1, + PieceOffset: 0, + PieceLength: commpCalc.PieceSize, + } + err = cl.AddDealForPiece(ctx, commpCalc.PieceCID, di) + require.NoError(t, err) + + // No pieces flagged, count and list of pieces should be empty + count, err := cl.FlaggedPiecesCount(ctx) + require.NoError(t, err) + require.Equal(t, 0, count) + + pcids, err := cl.FlaggedPiecesList(ctx, nil, 0, 10) + require.NoError(t, err) + require.Equal(t, 0, len(pcids)) + + // Flag a piece + err = cl.FlagPiece(ctx, commpCalc.PieceCID) + require.NoError(t, err) + + // Count and list of pieces should contain one piece + count, err = cl.FlaggedPiecesCount(ctx) + require.NoError(t, err) + require.Equal(t, 1, count) + + pcids, err = cl.FlaggedPiecesList(ctx, nil, 0, 10) + require.NoError(t, err) + require.Equal(t, 1, len(pcids)) + + // Unflag the piece + err = cl.UnflagPiece(ctx, commpCalc.PieceCID) + require.NoError(t, err) + + // Count and list of pieces should be empty + count, err = cl.FlaggedPiecesCount(ctx) + require.NoError(t, err) + require.Equal(t, 0, count) + + pcids, err = cl.FlaggedPiecesList(ctx, nil, 0, 10) + require.NoError(t, err) + require.Equal(t, 0, len(pcids)) +} diff --git a/piecedirectory/test_util.go b/piecedirectory/test_util.go new file mode 100644 index 000000000..f3aac7840 --- /dev/null +++ b/piecedirectory/test_util.go @@ -0,0 +1,117 @@ +package piecedirectory + +import ( + "context" + "io" + "testing" + "time" + + "github.com/filecoin-project/boost/piecedirectory/types" + mock_piecedirectory "github.com/filecoin-project/boost/piecedirectory/types/mocks" + "github.com/filecoin-project/boost/testutil" + "github.com/filecoin-project/boostd-data/couchbase" + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/go-commp-utils/writer" + "github.com/filecoin-project/go-state-types/abi" + "github.com/golang/mock/gomock" + "github.com/ipld/go-car/v2" + "github.com/stretchr/testify/require" +) + +var testCouchSettings = couchbase.DBSettings{ + ConnectString: "couchbase://localhost", + Auth: couchbase.DBSettingsAuth{ + Username: "Administrator", + Password: "boostdemo", + }, + PieceMetadataBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + MultihashToPiecesBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + PieceOffsetsBucket: couchbase.DBSettingsBucket{ + RAMQuotaMB: 128, + }, + TestMode: true, +} + +// Get the index records from the CAR file +func GetRecords(t *testing.T, reader car.SectionReader) []model.Record { + _, err := reader.Seek(0, io.SeekStart) + require.NoError(t, err) + + blockReader, err := car.NewBlockReader(reader) + require.NoError(t, err) + + var recs []model.Record + blockMetadata, err := blockReader.SkipNext() + for err == nil { + recs = append(recs, model.Record{ + Cid: blockMetadata.Cid, + OffsetSize: model.OffsetSize{ + Offset: blockMetadata.Offset, + Size: blockMetadata.Size, + }, + }) + + blockMetadata, err = blockReader.SkipNext() + } + require.ErrorIs(t, err, io.EOF) + + return recs +} + +func CreateCarFile(t *testing.T) string { + rseed := int(time.Now().UnixMilli()) + randomFilePath, err := testutil.CreateRandomFile(t.TempDir(), rseed, 64*1024) + require.NoError(t, err) + _, carFilePath, err := testutil.CreateDenseCARv2(t.TempDir(), randomFilePath) + require.NoError(t, err) + return carFilePath +} + +func CalculateCommp(t *testing.T, rdr io.ReadSeeker) writer.DataCIDSize { + _, err := rdr.Seek(0, io.SeekStart) + require.NoError(t, err) + + w := &writer.Writer{} + _, err = io.CopyBuffer(w, rdr, make([]byte, writer.CommPBuf)) + require.NoError(t, err) + + commp, err := w.Sum() + require.NoError(t, err) + + return commp +} + +func CreateMockPieceReader(t *testing.T, reader car.SectionReader) *mock_piecedirectory.MockPieceReader { + ctrl := gomock.NewController(t) + pr := mock_piecedirectory.NewMockPieceReader(ctrl) + pr.EXPECT().GetReader(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(_ context.Context, _ abi.SectorNumber, _ abi.PaddedPieceSize, _ abi.PaddedPieceSize) (types.SectionReader, error) { + _, err := reader.Seek(0, io.SeekStart) + return MockSectionReader{reader}, err + }) + return pr +} + +type MockSectionReader struct { + car.SectionReader +} + +func (MockSectionReader) Close() error { return nil } + +func CreateMockDoctorSealingApi() *mockSealingApi { + return &mockSealingApi{isUnsealed: true} +} + +type mockSealingApi struct { + isUnsealed bool +} + +var _ SealingApi = (*mockSealingApi)(nil) + +func (m *mockSealingApi) IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) { + return m.isUnsealed, nil +} diff --git a/piecedirectory/types/mocks/piecedirectory.go b/piecedirectory/types/mocks/piecedirectory.go new file mode 100644 index 000000000..70b61ff8e --- /dev/null +++ b/piecedirectory/types/mocks/piecedirectory.go @@ -0,0 +1,453 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/filecoin-project/boost/piecedirectory/types (interfaces: SectionReader,PieceReader,Store) + +// Package mock_piecedirectory is a generated GoMock package. +package mock_piecedirectory + +import ( + context "context" + reflect "reflect" + time "time" + + types "github.com/filecoin-project/boost/piecedirectory/types" + model "github.com/filecoin-project/boostd-data/model" + abi "github.com/filecoin-project/go-state-types/abi" + gomock "github.com/golang/mock/gomock" + cid "github.com/ipfs/go-cid" + index "github.com/ipld/go-car/v2/index" + multihash "github.com/multiformats/go-multihash" +) + +// MockSectionReader is a mock of SectionReader interface. +type MockSectionReader struct { + ctrl *gomock.Controller + recorder *MockSectionReaderMockRecorder +} + +// MockSectionReaderMockRecorder is the mock recorder for MockSectionReader. +type MockSectionReaderMockRecorder struct { + mock *MockSectionReader +} + +// NewMockSectionReader creates a new mock instance. +func NewMockSectionReader(ctrl *gomock.Controller) *MockSectionReader { + mock := &MockSectionReader{ctrl: ctrl} + mock.recorder = &MockSectionReaderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSectionReader) EXPECT() *MockSectionReaderMockRecorder { + return m.recorder +} + +// Close mocks base method. +func (m *MockSectionReader) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockSectionReaderMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSectionReader)(nil).Close)) +} + +// Read mocks base method. +func (m *MockSectionReader) Read(arg0 []byte) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Read", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Read indicates an expected call of Read. +func (mr *MockSectionReaderMockRecorder) Read(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Read", reflect.TypeOf((*MockSectionReader)(nil).Read), arg0) +} + +// ReadAt mocks base method. +func (m *MockSectionReader) ReadAt(arg0 []byte, arg1 int64) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReadAt", arg0, arg1) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ReadAt indicates an expected call of ReadAt. +func (mr *MockSectionReaderMockRecorder) ReadAt(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadAt", reflect.TypeOf((*MockSectionReader)(nil).ReadAt), arg0, arg1) +} + +// Seek mocks base method. +func (m *MockSectionReader) Seek(arg0 int64, arg1 int) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Seek", arg0, arg1) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Seek indicates an expected call of Seek. +func (mr *MockSectionReaderMockRecorder) Seek(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Seek", reflect.TypeOf((*MockSectionReader)(nil).Seek), arg0, arg1) +} + +// MockPieceReader is a mock of PieceReader interface. +type MockPieceReader struct { + ctrl *gomock.Controller + recorder *MockPieceReaderMockRecorder +} + +// MockPieceReaderMockRecorder is the mock recorder for MockPieceReader. +type MockPieceReaderMockRecorder struct { + mock *MockPieceReader +} + +// NewMockPieceReader creates a new mock instance. +func NewMockPieceReader(ctrl *gomock.Controller) *MockPieceReader { + mock := &MockPieceReader{ctrl: ctrl} + mock.recorder = &MockPieceReaderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPieceReader) EXPECT() *MockPieceReaderMockRecorder { + return m.recorder +} + +// GetReader mocks base method. +func (m *MockPieceReader) GetReader(arg0 context.Context, arg1 abi.SectorNumber, arg2, arg3 abi.PaddedPieceSize) (types.SectionReader, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetReader", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(types.SectionReader) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetReader indicates an expected call of GetReader. +func (mr *MockPieceReaderMockRecorder) GetReader(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetReader", reflect.TypeOf((*MockPieceReader)(nil).GetReader), arg0, arg1, arg2, arg3) +} + +// MockStore is a mock of Store interface. +type MockStore struct { + ctrl *gomock.Controller + recorder *MockStoreMockRecorder +} + +// MockStoreMockRecorder is the mock recorder for MockStore. +type MockStoreMockRecorder struct { + mock *MockStore +} + +// NewMockStore creates a new mock instance. +func NewMockStore(ctrl *gomock.Controller) *MockStore { + mock := &MockStore{ctrl: ctrl} + mock.recorder = &MockStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockStore) EXPECT() *MockStoreMockRecorder { + return m.recorder +} + +// AddDealForPiece mocks base method. +func (m *MockStore) AddDealForPiece(arg0 context.Context, arg1 cid.Cid, arg2 model.DealInfo) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddDealForPiece", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddDealForPiece indicates an expected call of AddDealForPiece. +func (mr *MockStoreMockRecorder) AddDealForPiece(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddDealForPiece", reflect.TypeOf((*MockStore)(nil).AddDealForPiece), arg0, arg1, arg2) +} + +// AddIndex mocks base method. +func (m *MockStore) AddIndex(arg0 context.Context, arg1 cid.Cid, arg2 []model.Record, arg3 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddIndex", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddIndex indicates an expected call of AddIndex. +func (mr *MockStoreMockRecorder) AddIndex(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddIndex", reflect.TypeOf((*MockStore)(nil).AddIndex), arg0, arg1, arg2, arg3) +} + +// FlagPiece mocks base method. +func (m *MockStore) FlagPiece(arg0 context.Context, arg1 cid.Cid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FlagPiece", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// FlagPiece indicates an expected call of FlagPiece. +func (mr *MockStoreMockRecorder) FlagPiece(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlagPiece", reflect.TypeOf((*MockStore)(nil).FlagPiece), arg0, arg1) +} + +// FlaggedPiecesCount mocks base method. +func (m *MockStore) FlaggedPiecesCount(arg0 context.Context) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FlaggedPiecesCount", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FlaggedPiecesCount indicates an expected call of FlaggedPiecesCount. +func (mr *MockStoreMockRecorder) FlaggedPiecesCount(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlaggedPiecesCount", reflect.TypeOf((*MockStore)(nil).FlaggedPiecesCount), arg0) +} + +// FlaggedPiecesList mocks base method. +func (m *MockStore) FlaggedPiecesList(arg0 context.Context, arg1 *time.Time, arg2, arg3 int) ([]model.FlaggedPiece, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FlaggedPiecesList", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].([]model.FlaggedPiece) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FlaggedPiecesList indicates an expected call of FlaggedPiecesList. +func (mr *MockStoreMockRecorder) FlaggedPiecesList(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FlaggedPiecesList", reflect.TypeOf((*MockStore)(nil).FlaggedPiecesList), arg0, arg1, arg2, arg3) +} + +// GetIndex mocks base method. +func (m *MockStore) GetIndex(arg0 context.Context, arg1 cid.Cid) (index.Index, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIndex", arg0, arg1) + ret0, _ := ret[0].(index.Index) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetIndex indicates an expected call of GetIndex. +func (mr *MockStoreMockRecorder) GetIndex(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIndex", reflect.TypeOf((*MockStore)(nil).GetIndex), arg0, arg1) +} + +// GetOffsetSize mocks base method. +func (m *MockStore) GetOffsetSize(arg0 context.Context, arg1 cid.Cid, arg2 multihash.Multihash) (*model.OffsetSize, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetOffsetSize", arg0, arg1, arg2) + ret0, _ := ret[0].(*model.OffsetSize) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetOffsetSize indicates an expected call of GetOffsetSize. +func (mr *MockStoreMockRecorder) GetOffsetSize(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOffsetSize", reflect.TypeOf((*MockStore)(nil).GetOffsetSize), arg0, arg1, arg2) +} + +// GetPieceDeals mocks base method. +func (m *MockStore) GetPieceDeals(arg0 context.Context, arg1 cid.Cid) ([]model.DealInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPieceDeals", arg0, arg1) + ret0, _ := ret[0].([]model.DealInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPieceDeals indicates an expected call of GetPieceDeals. +func (mr *MockStoreMockRecorder) GetPieceDeals(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPieceDeals", reflect.TypeOf((*MockStore)(nil).GetPieceDeals), arg0, arg1) +} + +// GetPieceMetadata mocks base method. +func (m *MockStore) GetPieceMetadata(arg0 context.Context, arg1 cid.Cid) (model.Metadata, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPieceMetadata", arg0, arg1) + ret0, _ := ret[0].(model.Metadata) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPieceMetadata indicates an expected call of GetPieceMetadata. +func (mr *MockStoreMockRecorder) GetPieceMetadata(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPieceMetadata", reflect.TypeOf((*MockStore)(nil).GetPieceMetadata), arg0, arg1) +} + +// IsCompleteIndex mocks base method. +func (m *MockStore) IsCompleteIndex(arg0 context.Context, arg1 cid.Cid) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsCompleteIndex", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsCompleteIndex indicates an expected call of IsCompleteIndex. +func (mr *MockStoreMockRecorder) IsCompleteIndex(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCompleteIndex", reflect.TypeOf((*MockStore)(nil).IsCompleteIndex), arg0, arg1) +} + +// IsIndexed mocks base method. +func (m *MockStore) IsIndexed(arg0 context.Context, arg1 cid.Cid) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsIndexed", arg0, arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsIndexed indicates an expected call of IsIndexed. +func (mr *MockStoreMockRecorder) IsIndexed(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsIndexed", reflect.TypeOf((*MockStore)(nil).IsIndexed), arg0, arg1) +} + +// ListPieces mocks base method. +func (m *MockStore) ListPieces(arg0 context.Context) ([]cid.Cid, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListPieces", arg0) + ret0, _ := ret[0].([]cid.Cid) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListPieces indicates an expected call of ListPieces. +func (mr *MockStoreMockRecorder) ListPieces(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPieces", reflect.TypeOf((*MockStore)(nil).ListPieces), arg0) +} + +// MarkIndexErrored mocks base method. +func (m *MockStore) MarkIndexErrored(arg0 context.Context, arg1 cid.Cid, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MarkIndexErrored", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// MarkIndexErrored indicates an expected call of MarkIndexErrored. +func (mr *MockStoreMockRecorder) MarkIndexErrored(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MarkIndexErrored", reflect.TypeOf((*MockStore)(nil).MarkIndexErrored), arg0, arg1, arg2) +} + +// NextPiecesToCheck mocks base method. +func (m *MockStore) NextPiecesToCheck(arg0 context.Context) ([]cid.Cid, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NextPiecesToCheck", arg0) + ret0, _ := ret[0].([]cid.Cid) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NextPiecesToCheck indicates an expected call of NextPiecesToCheck. +func (mr *MockStoreMockRecorder) NextPiecesToCheck(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NextPiecesToCheck", reflect.TypeOf((*MockStore)(nil).NextPiecesToCheck), arg0) +} + +// PiecesContainingMultihash mocks base method. +func (m *MockStore) PiecesContainingMultihash(arg0 context.Context, arg1 multihash.Multihash) ([]cid.Cid, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PiecesContainingMultihash", arg0, arg1) + ret0, _ := ret[0].([]cid.Cid) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PiecesContainingMultihash indicates an expected call of PiecesContainingMultihash. +func (mr *MockStoreMockRecorder) PiecesContainingMultihash(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PiecesContainingMultihash", reflect.TypeOf((*MockStore)(nil).PiecesContainingMultihash), arg0, arg1) +} + +// RemoveDealForPiece mocks base method. +func (m *MockStore) RemoveDealForPiece(arg0 context.Context, arg1 cid.Cid, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveDealForPiece", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveDealForPiece indicates an expected call of RemoveDealForPiece. +func (mr *MockStoreMockRecorder) RemoveDealForPiece(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveDealForPiece", reflect.TypeOf((*MockStore)(nil).RemoveDealForPiece), arg0, arg1, arg2) +} + +// RemoveIndexes mocks base method. +func (m *MockStore) RemoveIndexes(arg0 context.Context, arg1 cid.Cid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveIndexes", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveIndexes indicates an expected call of RemoveIndexes. +func (mr *MockStoreMockRecorder) RemoveIndexes(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveIndexes", reflect.TypeOf((*MockStore)(nil).RemoveIndexes), arg0, arg1) +} + +// RemovePieceMetadata mocks base method. +func (m *MockStore) RemovePieceMetadata(arg0 context.Context, arg1 cid.Cid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemovePieceMetadata", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemovePieceMetadata indicates an expected call of RemovePieceMetadata. +func (mr *MockStoreMockRecorder) RemovePieceMetadata(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemovePieceMetadata", reflect.TypeOf((*MockStore)(nil).RemovePieceMetadata), arg0, arg1) +} + +// SetCarSize mocks base method. +func (m *MockStore) SetCarSize(arg0 context.Context, arg1 cid.Cid, arg2 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetCarSize", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetCarSize indicates an expected call of SetCarSize. +func (mr *MockStoreMockRecorder) SetCarSize(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCarSize", reflect.TypeOf((*MockStore)(nil).SetCarSize), arg0, arg1, arg2) +} + +// UnflagPiece mocks base method. +func (m *MockStore) UnflagPiece(arg0 context.Context, arg1 cid.Cid) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UnflagPiece", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UnflagPiece indicates an expected call of UnflagPiece. +func (mr *MockStoreMockRecorder) UnflagPiece(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnflagPiece", reflect.TypeOf((*MockStore)(nil).UnflagPiece), arg0, arg1) +} diff --git a/piecedirectory/types/types.go b/piecedirectory/types/types.go new file mode 100644 index 000000000..8221dd5d3 --- /dev/null +++ b/piecedirectory/types/types.go @@ -0,0 +1,60 @@ +package types + +import ( + "context" + "errors" + "io" + "time" + + "github.com/filecoin-project/boostd-data/model" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/v2/index" + "github.com/multiformats/go-multihash" +) + +//go:generate go run github.com/golang/mock/mockgen -destination=mocks/piecedirectory.go -package=mock_piecedirectory . SectionReader,PieceReader,Store + +type SectionReader interface { + io.Reader + io.ReaderAt + io.Seeker + io.Closer +} + +var ErrSealed = errors.New("sector is not unsealed") + +type PieceReader interface { + // GetReader returns a reader over a piece. If there is no unsealed copy, returns ErrSealed. + GetReader(ctx context.Context, id abi.SectorNumber, offset abi.PaddedPieceSize, length abi.PaddedPieceSize) (SectionReader, error) +} + +type Store interface { + AddDealForPiece(ctx context.Context, pieceCid cid.Cid, dealInfo model.DealInfo) error + AddIndex(ctx context.Context, pieceCid cid.Cid, records []model.Record, isCompleteIndex bool) error + IsIndexed(ctx context.Context, pieceCid cid.Cid) (bool, error) + IsCompleteIndex(ctx context.Context, pieceCid cid.Cid) (bool, error) + GetIndex(ctx context.Context, pieceCid cid.Cid) (index.Index, error) + GetOffsetSize(ctx context.Context, pieceCid cid.Cid, hash multihash.Multihash) (*model.OffsetSize, error) + GetPieceMetadata(ctx context.Context, pieceCid cid.Cid) (model.Metadata, error) + ListPieces(ctx context.Context) ([]cid.Cid, error) + GetPieceDeals(ctx context.Context, pieceCid cid.Cid) ([]model.DealInfo, error) + SetCarSize(ctx context.Context, pieceCid cid.Cid, size uint64) error + PiecesContainingMultihash(ctx context.Context, m multihash.Multihash) ([]cid.Cid, error) + MarkIndexErrored(ctx context.Context, pieceCid cid.Cid, err string) error + RemoveDealForPiece(context.Context, cid.Cid, string) error + RemovePieceMetadata(context.Context, cid.Cid) error + RemoveIndexes(context.Context, cid.Cid) error + NextPiecesToCheck(ctx context.Context) ([]cid.Cid, error) + FlagPiece(ctx context.Context, pieceCid cid.Cid) error + UnflagPiece(ctx context.Context, pieceCid cid.Cid) error + FlaggedPiecesList(ctx context.Context, cursor *time.Time, offset int, limit int) ([]model.FlaggedPiece, error) + FlaggedPiecesCount(ctx context.Context) (int, error) +} + +// PieceDirMetadata has the db metadata info and a flag to indicate if this +// process is currently indexing the piece +type PieceDirMetadata struct { + model.Metadata + Indexing bool +} diff --git a/react/src/App.js b/react/src/App.js index 6b6c025f6..52ae7157c 100644 --- a/react/src/App.js +++ b/react/src/App.js @@ -17,8 +17,9 @@ import {Epoch} from "./Epoch"; import {LegacyDealDetail} from "./LegacyDealDetail" import {SettingsPage} from "./Settings"; import {Banner} from "./Banner"; +import {PieceDirectoryDetail, PieceDirectoryPage} from "./PieceDirectory"; import {ProposalLogsPage} from "./ProposalLogs"; -import {InspectPage} from "./Inspect"; +import {InspectPage, InspectPiecePage} from "./Inspect"; import {RetrievalLogsPage} from "./RetrievalLogs"; import {RetrievalLogDetail} from "./RetrievalLogDetail"; @@ -52,9 +53,12 @@ function App(props) { } /> } /> } /> + } /> } /> } /> } /> + } /> + } /> } /> } /> diff --git a/react/src/DealPublish.js b/react/src/DealPublish.js index f0f523e8e..98dff692a 100644 --- a/react/src/DealPublish.js +++ b/react/src/DealPublish.js @@ -17,7 +17,7 @@ export function DealPublishPage(props) { function DealPublishContent() { const {loading, error, data} = useQuery(DealPublishQuery, { - pollInterval: 5000, + pollInterval: 10000, }) const [publishNow] = useMutation(DealPublishNowMutation, { refetchQueries: [{ query: DealPublishQuery }] @@ -110,7 +110,7 @@ function DealsTable(props) { export function DealPublishMenuItem(props) { const {data} = useQuery(DealPublishQuery, { - pollInterval: 5000, + pollInterval: 10000, fetchPolicy: 'network-only', }) @@ -126,4 +126,4 @@ export function DealPublishMenuItem(props) { ) : null} ) -} \ No newline at end of file +} diff --git a/react/src/Deals.js b/react/src/Deals.js index 068d07617..2a032a27f 100644 --- a/react/src/Deals.js +++ b/react/src/Deals.js @@ -89,7 +89,7 @@ function StorageDealsContent(props) { const dealListOffset = (pageNum-1) * dealsPerPage const queryCursor = (pageNum === 1) ? null : params.cursor const {loading, error, data} = useQuery(DealsListQuery, { - pollInterval: searchQuery ? undefined : 1000, + pollInterval: searchQuery ? undefined : 10000, variables: { query: searchQuery, filter: searchFilters, @@ -166,7 +166,7 @@ function StorageDealsContent(props) { function LegacyDealsLink(props) { const {data} = useQuery(LegacyDealsCountQuery, { - pollInterval: 5000, + pollInterval: 10000, fetchPolicy: 'network-only', }) @@ -358,7 +358,7 @@ function TransferRate({deal}) { export function StorageDealsMenuItem(props) { const {data} = useQuery(DealsCountQuery, { - pollInterval: 5000, + pollInterval: 10000, fetchPolicy: 'network-only', }) diff --git a/react/src/Epoch.js b/react/src/Epoch.js index 2a50fe647..3ceb6faa2 100644 --- a/react/src/Epoch.js +++ b/react/src/Epoch.js @@ -6,7 +6,7 @@ import {Info} from "./Info"; export function Epoch(props) { const {data} = useQuery(EpochQuery, { - pollInterval: 5000, + pollInterval: 10000, fetchPolicy: "network-only", }) diff --git a/react/src/Funds.js b/react/src/Funds.js index e00da977f..8457da34f 100644 --- a/react/src/Funds.js +++ b/react/src/Funds.js @@ -23,7 +23,7 @@ export function FundsPage(props) { } function FundsChart(props) { - const {loading, error, data} = useQuery(FundsQuery, { pollInterval: 1000 }) + const {loading, error, data} = useQuery(FundsQuery, { pollInterval: 30000 }) if (loading) { return
Loading...
@@ -273,7 +273,7 @@ function FundsLogs(props) { } catch {} } const {loading, error, data} = useQuery(FundsLogsQuery, { - pollInterval: pageNum === 1 ? 5000 : undefined, + pollInterval: pageNum === 1 ? 10000 : undefined, variables: { cursor: queryCursor, limit: rowsPerPage, @@ -335,7 +335,7 @@ function FundsLog(props) { export function FundsMenuItem(props) { const {data} = useQuery(FundsQuery, { - pollInterval: 5000, + pollInterval: 10000, fetchPolicy: "network-only", }) @@ -393,4 +393,4 @@ export function FundsMenuItem(props) { {humanFIL(pubMsg.used)} of {humanFIL(pubMsg.total)} used -} \ No newline at end of file +} diff --git a/react/src/Info.css b/react/src/Info.css index e495ed90c..999b5b085 100644 --- a/react/src/Info.css +++ b/react/src/Info.css @@ -35,4 +35,13 @@ .info .content a { text-decoration: underline; -} \ No newline at end of file +} + +.info .info-item { + padding: 0.5em 0.5em 1em 0.5em; +} + +.info .info-item .info-title { + padding-bottom: 0.5em; + font-size: 1.125em; +} diff --git a/react/src/Info.js b/react/src/Info.js index f508c8644..98500fa69 100644 --- a/react/src/Info.js +++ b/react/src/Info.js @@ -26,3 +26,12 @@ function InfoBox(props) { ) } + +export function InfoListItem(props) { + return
+
{props.title}
+
+ {props.children} +
+
+} diff --git a/react/src/Inspect.css b/react/src/Inspect.css index a3c4180dc..6bf974f48 100644 --- a/react/src/Inspect.css +++ b/react/src/Inspect.css @@ -1,14 +1,14 @@ -.inspect table { +.inspect-content table { font-size: 1em; width: 100%; } -.inspect td, .inspect th { +.inspect-content td, .inspect-content th { padding: 0.5em 1em; font-weight: normal; } -.inspect th { +.inspect-content th { white-space: nowrap; text-align: left; opacity: 0.6; @@ -72,6 +72,7 @@ .piece-detail table th { text-align: left; + vertical-align: top; } .piece-detail table.deals .param { @@ -82,7 +83,7 @@ padding: 0.5em 1em; } -.inspect .title { +.inspect-content .title { font-size: 1.25em; margin: 2em 0.8em 1em 0.8em; } @@ -93,4 +94,13 @@ .inspect > p { padding: 2em 1em; -} \ No newline at end of file +} + +.flagged-pieces-none { + padding: 1em 1em 4em 1em; +} + +.button.build-index { + display: inline-block; + margin: 1em 0; +} diff --git a/react/src/Inspect.js b/react/src/Inspect.js index 633fd6305..155515b70 100644 --- a/react/src/Inspect.js +++ b/react/src/Inspect.js @@ -1,35 +1,185 @@ -import {useQuery} from "@apollo/react-hooks"; +import {useMutation, useQuery} from "@apollo/react-hooks"; import { + FlaggedPiecesQuery, PieceBuildIndexMutation, PieceStatusQuery, PiecesWithPayloadCidQuery, PiecesWithRootPayloadCidQuery } from "./gql"; import moment from "moment"; import {DebounceInput} from 'react-debounce-input'; import React, {useState} from "react"; import {PageContainer, ShortDealLink} from "./Components"; -import {Link, useParams} from "react-router-dom"; +import {Link, useNavigate, useParams} from "react-router-dom"; import {dateFormat} from "./util-date"; import xImg from './bootstrap-icons/icons/x-lg.svg' import inspectImg from './bootstrap-icons/icons/wrench.svg' import './Inspect.css' +import {Pagination} from "./Pagination"; +import {Info, InfoListItem} from "./Info"; + +var inspectBasePath = '/inspect' export function InspectMenuItem(props) { return ( - +

Inspect

) } +// Main page with flagged pieces export function InspectPage(props) { return } -function InspectContent(props) { +function InspectContent() { const params = useParams() const [searchQuery, setSearchQuery] = useState(params.query) + + const flaggedPiecesContent = searchQuery ? null : + + const showSearchPrompt = flaggedPiecesContent == null + return
+ { } + { flaggedPiecesContent } +
+} + +function FlaggedPieces({setSearchQuery}) { + const navigate = useNavigate() + const params = useParams() + const pageNum = (params.pageNum && parseInt(params.pageNum)) || 1 + + var [rowsPerPage, setRowsPerPage] = useState(RowsPerPage.load) + const onRowsPerPageChange = (e) => { + const val = parseInt(e.target.value) + RowsPerPage.save(val) + setRowsPerPage(val) + navigate(inspectBasePath) + scrollTop() + } + + // Fetch rows on this page + const listOffset = (pageNum-1) * rowsPerPage + const queryCursor = (pageNum === 1) ? null : params.cursor + const {loading, error, data} = useQuery(FlaggedPiecesQuery, { + pollInterval: 10000, + variables: { + cursor: queryCursor, + offset: listOffset, + limit: rowsPerPage, + }, + fetchPolicy: 'network-only', + }) + + if (error) return
Error: {error.message + " - check connection to Boost server"}
+ if (loading) return
Loading...
+ + var res = data.piecesFlagged + var rows = res.pieces + const totalCount = data.piecesFlagged.totalCount + const moreRows = data.piecesFlagged.more + + if (!totalCount) { + return
+ Boost doctor did not find any pieces with errors +
+ } + + var cursor = params.cursor + if (pageNum === 1 && rows.length) { + cursor = rows[0].CreatedAt.getTime() + } + + const paginationParams = { + basePath: inspectBasePath, + cursor, pageNum, totalCount, + rowsPerPage: rowsPerPage, + moreRows: moreRows, + onRowsPerPageChange: onRowsPerPageChange, + onLinkClick: scrollTop, + } + + return
+

Flagged pieces

+ + + + + + + + + + + {rows.map(piece => ( + + ))} + +
Piece CIDIndexUnsealed CopyDeals
+ + +
+} + +function FlaggedPieceRow({piece}) { + // Lookup the piece by piece CID. + // We do this asynchronously instead of as part of the list query so that + // checking for unseal status of each piece doesn't block the whole page. + const { loading, error, data } = useQuery(PieceStatusQuery, { + variables: { + pieceCid: piece.PieceCid, + }, + }) + + var isUnsealedMsg + if (loading) { + isUnsealedMsg = '...' + } else if (error) { + isUnsealedMsg = error.Message + } else if (data && data.pieceStatus) { + const isUnsealed = hasUnsealedCopy(data.pieceStatus) + isUnsealedMsg = isUnsealed ? 'Yes' : 'No' + } + + return + + + {piece.PieceCid} + + + {piece.IndexStatus.Status} + {isUnsealedMsg} + {piece.Deals.length} + +} + +function hasUnsealedCopy(piece) { + for (var dl of piece.Deals) { + if (dl.SealStatus.IsUnsealed) { + return true + } + } + return false +} + +// Page showing information about a particular piece +export function InspectPiecePage(props) { + const params = useParams() + + return +
+ +
+ +} + +function SearchResults({searchQuery, setSearchQuery, showSearchPrompt}) { const handleSearchQueryChange = (event) => { setSearchQuery(event.target.value) } @@ -76,6 +226,7 @@ function InspectContent(props) { // Lookup a piece by piece CID const pieceRes = useQuery(PieceStatusQuery, { + pollInterval: 10000, variables: { pieceCid: pieceCid, }, @@ -105,7 +256,9 @@ function InspectContent(props) { const showPayload = pieceCids.length > 1 var content = null if (!errorMsg && !pieceStatus && !showPayload) { - content =

Enter piece CID or payload CID into the search box

+ if (showSearchPrompt) { + content =

Enter piece CID or payload CID into the search box

+ } } else if (!showPayload && !showPieceStats) { content =

No piece found with piece CID or payload CID {pieceCid}

} else { @@ -115,7 +268,9 @@ function InspectContent(props) { } return
- + { setSearchQuery ? ( + + ) : null } { errorMsg ?
Error: {errorMsg}
: null} { content }
@@ -133,41 +288,70 @@ function PiecesWithPayload({payloadCid, pieceCids, setSearchQuery}) { } function PieceStatus({pieceCid, pieceStatus, searchQuery}) { + // Re-build index + const [buildIndex] = useMutation(PieceBuildIndexMutation, { + // refetchQueries: props.refetchQueries, + variables: {pieceCid: pieceCid} + }) + if (!pieceStatus) { return
No piece found with piece CID {pieceCid}
} const rootCid = pieceStatus.Deals.length ? pieceStatus.Deals[0].Deal.DealDataRoot : null - const searchIsPayloadCid = searchQuery && searchQuery != pieceCid && searchQuery != rootCid + const searchIsAnyCid = searchQuery && searchQuery != pieceCid && searchQuery != rootCid + const searchIsPieceCid = searchQuery && searchQuery == pieceCid + const searchIsRootCid = searchQuery && searchQuery == rootCid + const indexFailed = pieceStatus.IndexStatus.Status === 'Failed' + const indexRegistered = pieceStatus.IndexStatus.Status === 'Registered' + const canReIndex = (indexFailed || indexRegistered) && hasUnsealedCopy(pieceStatus) return
- {searchIsPayloadCid ? ( + {searchIsAnyCid ? ( - + ) : null} {rootCid ? ( - + ) : null} - + {searchIsPieceCid ? ( + + ) : ( + + )} - +
Searched CID (non-root){searchQuery}{searchQuery}
Data Root CID{rootCid} + { searchIsRootCid ? {rootCid} : rootCid } +
Piece CID{pieceCid}{pieceCid}{pieceCid}
Index Status{pieceStatus.IndexStatus.Status} + + {pieceStatus.IndexStatus.Status} + {indexFailed && pieceStatus.IndexStatus.Error ? ': ' + pieceStatus.IndexStatus.Error : '' } + + +
+ {canReIndex ? ( +
+ Re-index +
+ ) : null} +
-

Piece Store

+

Local Index Directory

{pieceStatus.PieceInfoDeals.length ? ( @@ -190,11 +374,11 @@ function PieceStatus({pieceCid, pieceStatus, searchQuery}) {
) : ( -

No data found in piece store for piece CID {pieceCid}

+

No deals found in Local Index Directory for piece CID {pieceCid}

)}

Deals

- {pieceStatus.PieceInfoDeals.length ? ( + {pieceStatus.Deals.length ? ( @@ -245,3 +429,42 @@ function SearchBox(props) { { props.value ? clear : null } } + +function IndexStatusInfo() { + return + + There was no information found for this piece CID in the Local Index Directory. + + + The piece has been added to the Local Index Directory but has not yet been indexed. + + + The piece is currently being indexed. + + + The piece has been indexed successfully. + + + There was an error indexing the piece. + + +} + +const RowsPerPage = { + Default: 10, + + settingsKey: "settings.flagged-pieces.per-page", + + load: () => { + const saved = localStorage.getItem(RowsPerPage.settingsKey) + return JSON.parse(saved) || RowsPerPage.Default + }, + + save: (val) => { + localStorage.setItem(RowsPerPage.settingsKey, JSON.stringify(val)); + } +} + +function scrollTop() { + window.scrollTo({ top: 0, behavior: "smooth" }) +} diff --git a/react/src/LegacyDealDetail.js b/react/src/LegacyDealDetail.js index 45739258b..290e14840 100644 --- a/react/src/LegacyDealDetail.js +++ b/react/src/LegacyDealDetail.js @@ -25,7 +25,7 @@ export function LegacyDealDetail(props) { const currentEpochData = useQuery(EpochQuery) const {loading, error, data} = useQuery(LegacyDealQuery, { - pollInterval: 1000, + pollInterval: 10000, variables: {id: params.dealID}, }) diff --git a/react/src/LegacyDeals.js b/react/src/LegacyDeals.js index a6b70ae5f..7439bcad6 100644 --- a/react/src/LegacyDeals.js +++ b/react/src/LegacyDeals.js @@ -58,7 +58,7 @@ function LegacyStorageDealsContent(props) { const dealListOffset = (pageNum-1) * dealsPerPage const queryCursor = pageNum === 1 ? null : params.cursor const {loading, error, data} = useQuery(LegacyDealsListQuery, { - pollInterval: searchQuery ? undefined : (pageNum === 1 ? 5000 : undefined), + pollInterval: searchQuery ? undefined : (pageNum === 1 ? 10000 : undefined), variables: { query: searchQuery, cursor: queryCursor, diff --git a/react/src/PieceDirectory.css b/react/src/PieceDirectory.css new file mode 100644 index 000000000..dd0189dd0 --- /dev/null +++ b/react/src/PieceDirectory.css @@ -0,0 +1,79 @@ +.piece-directory table { + font-size: 1em; + width: 100%; +} + +.piece-directory td, .piece-directory th { + padding: 0.5em 1em; + font-weight: normal; +} + +.piece-directory th { + white-space: nowrap; + text-align: left; + opacity: 0.6; +} + +.piece-directory .search { + position: absolute; + right: 8em; + top: 2em; +} + +.piece-directory .search input { + background-image: url("./bootstrap-icons/icons/search.svg"); + background-repeat: no-repeat; + background-position: left 0.5em center; + padding-left: 2em; + padding-right: 2em; +} + +.piece-directory .search .clear-text { + position: absolute; + right: 1em; + top: 1.1em; + cursor: pointer; +} + +.piece-directory h3 { + margin-top: 2em; +} + +.piece-directory .title { + font-size: 1.25em; + color: #a61e4d; + margin: 2em 0.8em 1.5em 0.8em; +} + +.piece-detail { + padding: 1em 0 5em 0; +} + +.piece-detail .title { + color: inherit; + font-weight: normal; + margin: 0.8em; +} + +.piece-detail table { + font-size: 1em; +} + +.piece-detail table.deals tr td:first-child { + width: 8em; +} +.piece-detail table.deals tr td:last-child { + line-break: anywhere; +} + +.piece-detail table th { + text-align: left; +} + +.piece-detail table.deals .param { + color: #777777; +} + +.piece-detail td, .piece-detail th { + padding: 0.5em 1em; +} diff --git a/react/src/PieceDirectory.js b/react/src/PieceDirectory.js new file mode 100644 index 000000000..656460ce6 --- /dev/null +++ b/react/src/PieceDirectory.js @@ -0,0 +1,279 @@ +import {useQuery} from "@apollo/react-hooks"; +import { + DealsCountQuery, +} from "./gql"; +import moment from "moment"; +import {DebounceInput} from 'react-debounce-input'; +import React, {useState} from "react"; +import {PageContainer, ShortDealLink} from "./Components"; +import {Link} from "react-router-dom"; +import {dateFormat} from "./util-date"; +import columnsGapImg from './bootstrap-icons/icons/columns-gap.svg' +import xImg from './bootstrap-icons/icons/x-lg.svg' +import './PieceDirectory.css' +import {addCommas} from "./util"; + +const piecesBasePath = '/pieces' + +export function PieceDirectoryPage(props) { + return + + +} + +function PieceDirectoryContent(props) { + const [searchQuery, setSearchQuery] = useState('') + const handleSearchQueryChange = (event) => { + setSearchQuery(event.target.value) + } + const clearSearchBox = () => { + setSearchQuery('') + } + + const pieceRes = searchPieceCid(searchQuery) + const payloadRes = searchPayloadCid(searchQuery) + + if (pieceRes.error || payloadRes.error) { + return
Error: {pieceRes.error ? pieceRes.error.message : payloadRes.error.message}
+ } + + if (pieceRes.loading || payloadRes.loading) { + return
Loading ...
+ } + + const showPieces = !!pieceRes.data.piece + const showPayload = !!payloadRes.data.payload.Pieces.length + const showInstructions = !showPieces && !showPayload + return
+ + + { showPieces ? : null } + { showPayload ? : null } + { showInstructions ?

Enter the piece CID or payload CID into the Search Box

: null } +
+} + +export function PieceDirectoryDetail({piece}) { + if (!piece) { + return
No piece found with piece CID {piece.PieceCid}
+ } + + return
+
+
+ + + + + + + + + + + + + + + + + + +
Piece CID{piece.PieceCid}
Data Root CID{piece.RootCid}
Indexed CIDs{addCommas(piece.CidCount)}
Index Status{piece.IndexStatus}
+ +
Deals
+ + + + + + + + + + + + {piece.Deals.map(deal => ( + + + + + + + + + + ))} + +
CreatedAtDeal UUIDChain Deal IDSector NumberPiece OffsetPiece LengthUnsealed
{moment().format(dateFormat)}{deal.ChainDealID}{deal.SectorID}{deal.PieceOffset}{deal.PieceLength}{deal.IsUnsealed ? 'Yes' : 'No'}
+
+
+} + +export function PayloadMetaDetail({payload}) { + if (payload.Pieces.length === 0) { + return
No piece found with payload CID {payload.RootCid}
+ } + + return
+
Pieces containing payload CID {payload.RootCid}
+ {payload.Pieces.map(piece => )} +
+} + +function SearchBox(props) { + return
+ + { props.value ? clear : null } +
+} + +export function PiecesMenuItem(props) { + const {data} = useQuery(DealsCountQuery, { + pollInterval: 10000, + fetchPolicy: 'network-only', + }) + + return ( +
+ + +

Pieces

+ +
+ ) +} + +function searchPayloadCid(payloadCid) { + const pieces = [] + for (const p of mockPieces) { + if (p.RootCid == payloadCid) { + pieces.push(p) + } + } + + return { + data: { + payload: { + RootCid: payloadCid, + Pieces: pieces + } + } + } +} + +function searchPieceCid(pieceCid) { + for (const p of mockPieces) { + if (p.PieceCid == pieceCid) { + return { + data: { piece: p } + } + } + } + + return { + data: { piece: null } + } +} + +const mockPieces = [{ + PieceCid: 'baga6ea4seaqgbkvk6apsnfbtvnbjabpu4tsv6ns45c36452fdhkqxznt3vlnedq', + RootCid: 'bafybeiagwnqiviaae5aet2zivwhhsorg75x2wka2pu55o7grr23ulx5kxm', + CidCount: '3214', + IndexStatus: 'Complete', + Deals: [{ + DealUUID: '237fb3d0-fc11-40dc-a77a-0b75363ffa5e', + ChainDealID: '1', + SectorID: 1, + PieceLength: 2048, + PieceOffset: 0, + IsUnsealed: true, + }, { + DealUUID: '25332be9-54b4-471e-a669-0757f6b61fe8', + ChainDealID: '2', + SectorID: 4, + PieceLength: 2048, + PieceOffset: 2048, + IsUnsealed: false, + }] +}, { + PieceCid: 'bafybeig3yg2msah74sgvow25uxddqbabex3f3mh6hysess3w5kmgiv6zqy', + RootCid: 'bafybeiagwnqiviaae5aet2zivwhhsorg75x2wka2pu55o7grr23ulx5kxm', + CidCount: '53234', + IndexStatus: 'Indexing', + Deals: [{ + DealUUID: '237fb3d0-fc11-40dc-a77a-0b75363ffa5e', + ChainDealID: '1', + SectorID: 1, + PieceLength: 2048, + PieceOffset: 0, + IsUnsealed: true, + }, { + DealUUID: '25332be9-54b4-471e-a669-0757f6b61fe8', + ChainDealID: '2', + SectorID: 4, + PieceLength: 2048, + PieceOffset: 2048, + IsUnsealed: false, + }] +}, { + PieceCid: 'bafybeiaij7wqrolyv5gx2glkordkq22yacpgg23bdwyweenwlknk37zjyu', + RootCid: 'bafybeigbb6jrtwwdlqtqu23uzvsezm5m7qpw57emqipy5muclgp5dakpmq', + CidCount: '14921', + IndexStatus: 'Complete', + Deals: [{ + DealUUID: '237fb3d0-fc11-40dc-a77a-0b75363ffa5e', + ChainDealID: '1', + SectorID: 1, + PieceLength: 2048, + PieceOffset: 0, + IsUnsealed: true, + }, { + DealUUID: '25332be9-54b4-471e-a669-0757f6b61fe8', + ChainDealID: '2', + SectorID: 4, + PieceLength: 2048, + PieceOffset: 2048, + IsUnsealed: false, + }] +}, { + PieceCid: 'bafybeicl6ujc6ncfktctxxroxognfn7d2fqavvrryoc2lv6m4i6hpbkfti', + Deals: [{ + DealUUID: '237fb3d0-fc11-40dc-a77a-0b75363ffa5e', + ChainDealID: '1', + SectorID: 1, + PieceLength: 2048, + PieceOffset: 0, + IsUnsealed: true, + }, { + DealUUID: '25332be9-54b4-471e-a669-0757f6b61fe8', + ChainDealID: '2', + SectorID: 4, + PieceLength: 2048, + PieceOffset: 2048, + IsUnsealed: true, + }, { + DealUUID: '237fb3d0-fc11-40dc-a77a-0b75363ffa5e', + ChainDealID: '3', + SectorID: 6, + PieceLength: 2048, + PieceOffset: 0, + IsUnsealed: false, + }] +}, { + PieceCid: 'bafybeihhponjv2pdbmg33nxvccrgj34546avahl3nojk5cplawofvn2d3m', + Deals: [{ + DealUUID: '25332be9-54b4-471e-a669-0757f6b61fe8', + ChainDealID: '2', + SectorID: 4, + PieceLength: 2048, + PieceOffset: 2048, + IsUnsealed: true, + }] +}] diff --git a/react/src/ProposalLogs.js b/react/src/ProposalLogs.js index 9adb156ad..361491d24 100644 --- a/react/src/ProposalLogs.js +++ b/react/src/ProposalLogs.js @@ -53,7 +53,7 @@ function ProposalLogsContent(props) { } catch {} } const {loading, error, data} = useQuery(ProposalLogsListQuery, { - pollInterval: 1000, + pollInterval: 10000, variables: { cursor: queryCursor, offset: listOffset, @@ -172,7 +172,7 @@ function LogRow(props) { export function ProposalLogsMenuItem(props) { const {data} = useQuery(ProposalLogsCountQuery, { - pollInterval: 5000, + pollInterval: 10000, fetchPolicy: 'network-only', variables: { accepted: true, diff --git a/react/src/RetrievalLogDetail.js b/react/src/RetrievalLogDetail.js index 1bde97a3e..b5a80acba 100644 --- a/react/src/RetrievalLogDetail.js +++ b/react/src/RetrievalLogDetail.js @@ -39,7 +39,7 @@ export function RetrievalLogDetail(props) { } const {loading, error, data} = useQuery(RetrievalLogQuery, { - pollInterval: 1000, + pollInterval: 10000, variables: { peerID: params.peerID, transferID: params.transferID, diff --git a/react/src/RetrievalLogs.js b/react/src/RetrievalLogs.js index f2197c877..116b1ba58 100644 --- a/react/src/RetrievalLogs.js +++ b/react/src/RetrievalLogs.js @@ -53,7 +53,7 @@ function RetrievalLogsContent(props) { } catch {} } const {loading, error, data} = useQuery(RetrievalLogsListQuery, { - pollInterval: 1000, + pollInterval: 10000, variables: { cursor: queryCursor, offset: listOffset, @@ -195,7 +195,7 @@ function TableRow(props) { export function RetrievalLogsMenuItem(props) { const {data} = useQuery(RetrievalLogsCountQuery, { - pollInterval: 5000, + pollInterval: 10000, fetchPolicy: 'network-only', variables: { accepted: true, diff --git a/react/src/SealingPipeline.js b/react/src/SealingPipeline.js index 37f314c0e..b4105b8f3 100644 --- a/react/src/SealingPipeline.js +++ b/react/src/SealingPipeline.js @@ -17,7 +17,7 @@ export function SealingPipelinePage(props) { } function SealingPipelineContent(props) { - const {loading, error, data} = useQuery(SealingPipelineQuery, { pollInterval: 2000 }) + const {loading, error, data} = useQuery(SealingPipelineQuery, { pollInterval: 10000 }) if (loading) { return
Loading...
@@ -194,7 +194,7 @@ function Workers(props) { export function SealingPipelineMenuItem(props) { const {data} = useQuery(SealingPipelineQuery, { - pollInterval: 5000, + pollInterval: 10000, fetchPolicy: "network-only", }) diff --git a/react/src/StorageSpace.js b/react/src/StorageSpace.js index 055a4e470..de13fa98e 100644 --- a/react/src/StorageSpace.js +++ b/react/src/StorageSpace.js @@ -17,7 +17,7 @@ export function StorageSpacePage(props) { } function StorageSpaceContent(props) { - const {loading, error, data} = useQuery(StorageQuery, { pollInterval: 1000 }) + const {loading, error, data} = useQuery(StorageQuery, { pollInterval: 10000 }) if (loading) { return
Loading...
@@ -82,7 +82,7 @@ function StorageSpaceContent(props) { } function LegacyStorageSpaceContent(props) { - const {loading, error, data} = useQuery(LegacyStorageQuery, { pollInterval: 1000 }) + const {loading, error, data} = useQuery(LegacyStorageQuery, { pollInterval: 10000 }) if (loading) { return
Loading...
@@ -138,7 +138,7 @@ function LegacyStorageSpaceContent(props) { export function StorageSpaceMenuItem(props) { const {data} = useQuery(StorageQuery, { - pollInterval: 5000, + pollInterval: 10000, fetchPolicy: 'network-only', }) @@ -172,4 +172,4 @@ export function StorageSpaceMenuItem(props) { ) -} \ No newline at end of file +} diff --git a/react/src/gql.js b/react/src/gql.js index 58377b889..33470f418 100644 --- a/react/src/gql.js +++ b/react/src/gql.js @@ -396,6 +396,39 @@ const PiecesWithPayloadCidQuery = gql` } `; +const FlaggedPiecesQuery = gql` + query AppFlaggedPiecesQuery($cursor: BigInt, $offset: Int, $limit: Int) { + piecesFlagged(cursor: $cursor, offset: $offset, limit: $limit) { + pieces { + CreatedAt + Piece { + PieceCid + IndexStatus { + Status + Error + } + Deals { + Deal { + ID + IsLegacy + CreatedAt + DealDataRoot + } + } + } + } + totalCount + more + } + } +`; + +const PieceBuildIndexMutation = gql` + mutation AppPieceBuildIndexMutation($pieceCid: String!) { + pieceBuildIndex(pieceCid: $pieceCid) + } +`; + const PieceStatusQuery = gql` query AppPieceStatusQuery($pieceCid: String!) { pieceStatus(pieceCid: $pieceCid) { @@ -675,7 +708,9 @@ export { RetrievalLogsCountQuery, PiecesWithRootPayloadCidQuery, PiecesWithPayloadCidQuery, + PieceBuildIndexMutation, PieceStatusQuery, + FlaggedPiecesQuery, StorageQuery, LegacyStorageQuery, FundsQuery, diff --git a/storagemarket/deal_execution.go b/storagemarket/deal_execution.go index 708528e35..34e3f347f 100644 --- a/storagemarket/deal_execution.go +++ b/storagemarket/deal_execution.go @@ -13,9 +13,7 @@ import ( "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints" "github.com/filecoin-project/boost/transport" transporttypes "github.com/filecoin-project/boost/transport/types" - "github.com/filecoin-project/dagstore" - "github.com/filecoin-project/go-fil-markets/piecestore" - "github.com/filecoin-project/go-fil-markets/stores" + "github.com/filecoin-project/boostd-data/model" "github.com/filecoin-project/go-padreader" "github.com/filecoin-project/go-state-types/abi" acrypto "github.com/filecoin-project/go-state-types/crypto" @@ -109,11 +107,11 @@ func (p *Provider) execDeal(deal *smtypes.ProviderDealState, dh *dealHandler) (d } deal.NBytesReceived = fi.Size() p.dealLogger.Infow(deal.DealUuid, "size of "+transferType, "filepath", deal.InboundFilePath, "size", fi.Size()) - } else { - // if the deal has already been handed to the sealer, the inbound file - // could already have been removed and in that case, the number of - // bytes received should be the same as deal size as we've already - // verified the transfer. + } else if !deal.IsOffline { + // For online deals where the deal has already been handed to the sealer, + // the inbound file could already have been removed and in that case, + // the number of bytes received should be the same as deal size as + // we've already verified the transfer. deal.NBytesReceived = int64(deal.Transfer.Size) } @@ -601,43 +599,23 @@ func (p *Provider) addPiece(ctx context.Context, pub event.Emitter, deal *types. } func (p *Provider) indexAndAnnounce(ctx context.Context, pub event.Emitter, deal *types.ProviderDealState) *dealMakingError { + // add deal to piece metadata store pc := deal.ClientDealProposal.Proposal.PieceCID - propCid, err := deal.ClientDealProposal.Proposal.Cid() - if err != nil { - return &dealMakingError{ - retry: types.DealRetryFatal, - error: fmt.Errorf("index and announce: getting deal proposal cid: %w", err), - } - } - - // add deal to piecestore - if err := p.ps.AddDealForPiece(pc, propCid, piecestore.DealInfo{ - DealID: deal.ChainDealID, - SectorID: deal.SectorID, - Offset: deal.Offset, - Length: deal.Length, + if err := p.piecedirectory.AddDealForPiece(ctx, pc, model.DealInfo{ + DealUuid: deal.DealUuid.String(), + ChainDealID: deal.ChainDealID, + MinerAddr: p.Address, + SectorID: deal.SectorID, + PieceOffset: deal.Offset, + PieceLength: deal.Length, + CarLength: uint64(deal.NBytesReceived), }); err != nil { return &dealMakingError{ retry: types.DealRetryAuto, - error: fmt.Errorf("failed to add deal to piecestore: %w", err), - } - } - p.dealLogger.Infow(deal.DealUuid, "deal successfully added to piecestore") - - // register with dagstore - err = stores.RegisterShardSync(ctx, p.dagst, pc, "", true) - - if err != nil { - if !errors.Is(err, dagstore.ErrShardExists) { - return &dealMakingError{ - retry: types.DealRetryAuto, - error: fmt.Errorf("failed to register deal with dagstore: %w", err), - } + error: fmt.Errorf("failed to add deal to piece metadata store: %w", err), } - p.dealLogger.Infow(deal.DealUuid, "deal has previously been registered in dagstore") - } else { - p.dealLogger.Infow(deal.DealUuid, "deal has successfully been registered in the dagstore") } + p.dealLogger.Infow(deal.DealUuid, "deal successfully added to piece metadata store") // if the index provider is enabled if p.ip.Enabled() { diff --git a/storagemarket/provider.go b/storagemarket/provider.go index 9d398515a..ca11e4770 100644 --- a/storagemarket/provider.go +++ b/storagemarket/provider.go @@ -17,6 +17,7 @@ import ( "github.com/filecoin-project/boost/fundmanager" "github.com/filecoin-project/boost/markets/utils" "github.com/filecoin-project/boost/node/modules/dtypes" + "github.com/filecoin-project/boost/piecedirectory" "github.com/filecoin-project/boost/storagemanager" "github.com/filecoin-project/boost/storagemarket/logs" "github.com/filecoin-project/boost/storagemarket/sealingpipeline" @@ -26,10 +27,8 @@ import ( "github.com/filecoin-project/boost/transport" "github.com/filecoin-project/boostd-data/shared/tracing" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/shared" "github.com/filecoin-project/go-fil-markets/storagemarket" - "github.com/filecoin-project/go-fil-markets/stores" lapi "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/api/v1api" ctypes "github.com/filecoin-project/lotus/chain/types" @@ -127,18 +126,16 @@ type Provider struct { dealLogger *logs.DealLogger - dagst stores.DAGStoreWrapper - ps piecestore.PieceStore - - ip types.IndexProvider - askGetter types.AskGetter - sigVerifier types.SignatureVerifier + piecedirectory *piecedirectory.PieceDirectory + ip types.IndexProvider + askGetter types.AskGetter + sigVerifier types.SignatureVerifier } func NewProvider(cfg Config, sqldb *sql.DB, dealsDB *db.DealsDB, fundMgr *fundmanager.FundManager, storageMgr *storagemanager.StorageManager, fullnodeApi v1api.FullNode, dp types.DealPublisher, addr address.Address, pa types.PieceAdder, commpCalc smtypes.CommpCalculator, sps sealingpipeline.API, cm types.ChainDealManager, df dtypes.StorageDealFilter, logsSqlDB *sql.DB, logsDB *db.LogsDB, - dagst stores.DAGStoreWrapper, ps piecestore.PieceStore, ip types.IndexProvider, askGetter types.AskGetter, + piecedirectory *piecedirectory.PieceDirectory, ip types.IndexProvider, askGetter types.AskGetter, sigVerifier types.SignatureVerifier, dl *logs.DealLogger, tspt transport.Transport) (*Provider, error) { xferLimiter, err := newTransferLimiter(cfg.TransferLimiter) @@ -198,12 +195,10 @@ func NewProvider(cfg Config, sqldb *sql.DB, dealsDB *db.DealsDB, fundMgr *fundma dealLogger: dl, logsDB: logsDB, - dagst: dagst, - ps: ps, - - ip: ip, - askGetter: askGetter, - sigVerifier: sigVerifier, + piecedirectory: piecedirectory, + ip: ip, + askGetter: askGetter, + sigVerifier: sigVerifier, }, nil } diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 62955902f..826d0242f 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -20,6 +20,8 @@ import ( "github.com/filecoin-project/boost/db" "github.com/filecoin-project/boost/fundmanager" + "github.com/filecoin-project/boost/piecedirectory" + mock_piecedirectory "github.com/filecoin-project/boost/piecedirectory/types/mocks" "github.com/filecoin-project/boost/storagemanager" "github.com/filecoin-project/boost/storagemarket/dealfilter" "github.com/filecoin-project/boost/storagemarket/logs" @@ -33,7 +35,6 @@ import ( "github.com/filecoin-project/boost/transport/mocks" tspttypes "github.com/filecoin-project/boost/transport/types" "github.com/filecoin-project/go-address" - piecestoreimpl "github.com/filecoin-project/go-fil-markets/piecestore/impl" "github.com/filecoin-project/go-fil-markets/shared_testutil" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-state-types/abi" @@ -50,8 +51,6 @@ import ( "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" carv2 "github.com/ipld/go-car/v2" "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/host" @@ -1147,12 +1146,12 @@ func (h *ProviderHarness) AssertPieceAdded(t *testing.T, ctx context.Context, dp // Assert that the original file data we sent matches what was sent to the sealer h.AssertSealedContents(t, carv2FilePath, *so.SealedBytes) // assert that dagstore and piecestore have this deal - dbState, err := h.DealsDB.ByID(ctx, dp.DealUUID) + _, err := h.DealsDB.ByID(ctx, dp.DealUUID) require.NoError(t, err) - rg, ok := h.DAGStore.GetRegistration(dbState.ClientDealProposal.Proposal.PieceCID) - require.True(t, ok) - require.True(t, rg.EagerInit) - require.Empty(t, rg.CarPath) + //rg, ok := h.DAGStore.GetRegistration(dbState.ClientDealProposal.Proposal.PieceCID) + //require.True(t, ok) + //require.True(t, rg.EagerInit) + //require.Empty(t, rg.CarPath) } func (h *ProviderHarness) AssertDealIndexed(t *testing.T, ctx context.Context, dp *types.DealParams, so *smtestutil.StubbedMinerOutput) { @@ -1517,13 +1516,18 @@ func NewHarness(t *testing.T, opts ...harnessOpt) *ProviderHarness { df = pc.dealFilter } - ps, err := piecestoreimpl.NewPieceStore(dssync.MutexWrap(ds.NewMapDatastore())) - require.NoError(t, err) - dagStore := shared_testutil.NewMockDagStoreWrapper(ps, nil) - askStore := &mockAskStore{} askStore.SetAsk(pc.price, pc.verifiedPrice, pc.minPieceSize, pc.maxPieceSize) + store := mock_piecedirectory.NewMockStore(ctrl) + store.EXPECT().IsIndexed(gomock.Any(), gomock.Any()).Return(true, nil).AnyTimes() + store.EXPECT().AddDealForPiece(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + + pdctx, cancel := context.WithCancel(context.Background()) + pm := piecedirectory.NewPieceDirectory(store, nil, 1) + pm.Start(pdctx) + t.Cleanup(cancel) + prvCfg := Config{ MaxTransferDuration: time.Hour, RemoteCommp: !pc.localCommp, @@ -1536,7 +1540,7 @@ func NewHarness(t *testing.T, opts ...harnessOpt) *ProviderHarness { StorageFilter: "1", } prov, err := NewProvider(prvCfg, sqldb, dealsDB, fm, sm, fn, minerStub, minerAddr, minerStub, minerStub, sps, minerStub, df, sqldb, - logsDB, dagStore, ps, minerStub, askStore, &mockSignatureVerifier{true, nil}, dl, tspt) + logsDB, pm, minerStub, askStore, &mockSignatureVerifier{true, nil}, dl, tspt) require.NoError(t, err) ph.Provider = prov @@ -1564,7 +1568,6 @@ func NewHarness(t *testing.T, opts ...harnessOpt) *ProviderHarness { secInfo := lapi.SectorInfo{State: lapi.SectorState(sealing.Proving)} ph.MockSealingPipelineAPI.EXPECT().SectorsStatus(gomock.Any(), gomock.Any(), false).Return(secInfo, nil).AnyTimes() - ph.DAGStore = dagStore ph.MockFullNode = fn return ph @@ -1594,7 +1597,7 @@ func (h *ProviderHarness) shutdownAndCreateNewProvider(t *testing.T, opts ...har // construct a new provider with pre-existing state prov, err := NewProvider(h.Provider.config, h.Provider.db, h.Provider.dealsDB, h.Provider.fundManager, h.Provider.storageManager, h.Provider.fullnodeApi, h.MinerStub, h.MinerAddr, h.MinerStub, h.MinerStub, h.MockSealingPipelineAPI, h.MinerStub, - df, h.Provider.logsSqlDB, h.Provider.logsDB, h.Provider.dagst, h.Provider.ps, h.MinerStub, h.Provider.askGetter, + df, h.Provider.logsSqlDB, h.Provider.logsDB, h.Provider.piecedirectory, h.MinerStub, h.Provider.askGetter, h.Provider.sigVerifier, h.Provider.dealLogger, h.Provider.Transport) require.NoError(t, err) @@ -1602,14 +1605,6 @@ func (h *ProviderHarness) shutdownAndCreateNewProvider(t *testing.T, opts ...har } func (h *ProviderHarness) Start(t *testing.T, ctx context.Context) { - require.NoError(t, h.Provider.ps.Start(ctx)) - ready := make(chan error) - h.Provider.ps.OnReady(func(err error) { - ready <- err - }) - - require.NoError(t, <-ready) - h.NormalServer.Start() h.BlockingServer.Start() h.DisconnectingServer.Start()