From 961657bfe8d1f67eb7e1c36b32c36adce594ad53 Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Tue, 16 May 2023 22:12:02 +0400 Subject: [PATCH 01/15] fix statx output string (#1451) --- cmd/boostx/stats_cmd.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/boostx/stats_cmd.go b/cmd/boostx/stats_cmd.go index 81ddced69..1b8775955 100644 --- a/cmd/boostx/stats_cmd.go +++ b/cmd/boostx/stats_cmd.go @@ -2,13 +2,14 @@ package main import ( "fmt" - "github.com/filecoin-project/boost/retrievalmarket/lp2pimpl" - transports_types "github.com/filecoin-project/boost/retrievalmarket/types" "regexp" "sort" "strings" "sync" + "github.com/filecoin-project/boost/retrievalmarket/lp2pimpl" + transports_types "github.com/filecoin-project/boost/retrievalmarket/types" + clinode "github.com/filecoin-project/boost/cli/node" "github.com/filecoin-project/boost/cmd" "github.com/filecoin-project/boostd-data/shared/cliutil" @@ -223,7 +224,7 @@ var statsCmd = &cli.Command{ fmt.Println("Total Boost nodes:", boostNodes) fmt.Println("Total Boost raw power:", boostRawBytePower) fmt.Println("Total Boost quality adj power:", boostQualityAdjPower) - fmt.Println("Total Lotus Markets nodes:", marketsNodes) + fmt.Println("Total Markets nodes:", marketsNodes) fmt.Println("Total SPs with minimum power: ", len(withMinPower)) fmt.Println("Total Indexer nodes:", indexerNodes) From a4b95bb5ea589b0a8edebbf8f1eb8377b9d40bd2 Mon Sep 17 00:00:00 2001 From: dirkmc Date: Fri, 19 May 2023 15:47:01 +0200 Subject: [PATCH 02/15] fix: flaky TestMultipleDealsConcurrent (#1458) --- storagemarket/provider_test.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/storagemarket/provider_test.go b/storagemarket/provider_test.go index 0ccafd7dd..2ba9f161c 100644 --- a/storagemarket/provider_test.go +++ b/storagemarket/provider_test.go @@ -10,7 +10,6 @@ import ( "math/rand" "net/http/httptest" "os" - "path" "path/filepath" "strings" "sync" @@ -1869,13 +1868,13 @@ func (ph *ProviderHarness) newDealBuilder(t *testing.T, seed int, opts ...dealPr require.NoError(tbuilder.t, err) carData, err := io.ReadAll(r) require.NoError(tbuilder.t, err) - carv1Path := path.Join(tbuilder.ph.TempDir, "v1.car") - err = os.WriteFile(carv1Path, carData, 0644) + carv1File, err := os.CreateTemp(tbuilder.ph.TempDir, "v1.car") require.NoError(tbuilder.t, err) - rd, err := carv2.OpenReader(carv1Path) + _, err = carv1File.Write(carData) require.NoError(tbuilder.t, err) - defer rd.Close() - carFilePath = carv1Path + err = carv1File.Close() + require.NoError(tbuilder.t, err) + carFilePath = carv1File.Name() } // generate CommP of the CARv2 file From 1a7b1933b44c1858912855dd878077d2098419ca Mon Sep 17 00:00:00 2001 From: dirkmc Date: Tue, 23 May 2023 10:40:46 +0200 Subject: [PATCH 03/15] Add option to serve index provider ads over http (#1452) * feat: option to serve index provider ads over http * fix: config naming, hostname parsing * fix: update docsgen * fix: log announce address * feat: add config for indexer direct announce urls * refactor: always announce over pubsub * fix: docsgen * test: add test case for empty announce address hostname * Add `boostd index announce-latest` command (#1456) * feat: boostd index announce-latest * feat: add announce-latest-http command * fix: default direct announce url * feat: update to index-provider v0.11.2 --- api/api.go | 2 + api/proxy_gen.go | 26 +++++++ build/openrpc/boost.json.gz | Bin 5503 -> 5619 bytes cmd/boostd/index.go | 54 +++++++++++++++ cmd/boostd/init.go | 8 ++- documentation/en/api-v1-methods.md | 37 ++++++++++ go.mod | 2 +- go.sum | 4 +- indexprovider/wrapper.go | 31 +++++++++ node/config/def.go | 13 +++- node/config/doc_gen.go | 98 ++++++++++++++++++++++++++- node/config/types.go | 58 +++++++++++++++- node/impl/boost.go | 8 +++ node/modules/storageminer_idxprov.go | 40 ++++++++--- util/addr.go | 26 +++++++ util/addr_test.go | 44 ++++++++++++ 16 files changed, 436 insertions(+), 15 deletions(-) create mode 100644 util/addr.go create mode 100644 util/addr_test.go diff --git a/api/api.go b/api/api.go index cc067d49c..6b803b8c6 100644 --- a/api/api.go +++ b/api/api.go @@ -35,6 +35,8 @@ type Boost interface { // MethodGroup: Boost BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin + BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) //perm:admin + BoostIndexerAnnounceLatestHttp(ctx context.Context, urls []string) (cid.Cid, error) //perm:admin BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*ProviderDealRejectionInfo, error) //perm:admin BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error) //perm:admin BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error) //perm:admin diff --git a/api/proxy_gen.go b/api/proxy_gen.go index b51ec916a..5121b8212 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -69,6 +69,10 @@ type BoostStruct struct { BoostIndexerAnnounceAllDeals func(p0 context.Context) error `perm:"admin"` + BoostIndexerAnnounceLatest func(p0 context.Context) (cid.Cid, error) `perm:"admin"` + + BoostIndexerAnnounceLatestHttp func(p0 context.Context, p1 []string) (cid.Cid, error) `perm:"admin"` + BoostMakeDeal func(p0 context.Context, p1 smtypes.DealParams) (*ProviderDealRejectionInfo, error) `perm:"write"` BoostOfflineDealWithData func(p0 context.Context, p1 uuid.UUID, p2 string, p3 bool) (*ProviderDealRejectionInfo, error) `perm:"admin"` @@ -444,6 +448,28 @@ func (s *BoostStub) BoostIndexerAnnounceAllDeals(p0 context.Context) error { return ErrNotSupported } +func (s *BoostStruct) BoostIndexerAnnounceLatest(p0 context.Context) (cid.Cid, error) { + if s.Internal.BoostIndexerAnnounceLatest == nil { + return *new(cid.Cid), ErrNotSupported + } + return s.Internal.BoostIndexerAnnounceLatest(p0) +} + +func (s *BoostStub) BoostIndexerAnnounceLatest(p0 context.Context) (cid.Cid, error) { + return *new(cid.Cid), ErrNotSupported +} + +func (s *BoostStruct) BoostIndexerAnnounceLatestHttp(p0 context.Context, p1 []string) (cid.Cid, error) { + if s.Internal.BoostIndexerAnnounceLatestHttp == nil { + return *new(cid.Cid), ErrNotSupported + } + return s.Internal.BoostIndexerAnnounceLatestHttp(p0, p1) +} + +func (s *BoostStub) BoostIndexerAnnounceLatestHttp(p0 context.Context, p1 []string) (cid.Cid, error) { + return *new(cid.Cid), ErrNotSupported +} + func (s *BoostStruct) BoostMakeDeal(p0 context.Context, p1 smtypes.DealParams) (*ProviderDealRejectionInfo, error) { if s.Internal.BoostMakeDeal == nil { return nil, ErrNotSupported diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index 5bcd3ae20f834fd87c8060988d0e1441745aa581..0680625fed6b2d7b49e393813eea898a42a62e38 100644 GIT binary patch literal 5619 zcmV`+$hU>HaWMe6%p_Bjpj^s!DkZjp;N(tU&AaSJQBk8=QbM&Y?F2tT|>SubV)ol%QLq~Y( z=x2{QMbw7+nSS9oglad}L+$)}q#OD@^aye=?{VDfz4FYCzBcp)q;tn2`q`&P9fJky z`8%?q=^)%XH>u-|A^#hFhq|F#keD8FsSoJOmoGEtGoCK7sl6cDOa96+^_C_*&{CaW zxNXgJFoh4aHT3^AG`FJx%km&0t#kk11MLNvpw_0Jp+V1cyaVm=GyY7`R3ohsrqIK{ zPUENj_M3*0?YI0)JxG@x)?Q-xEd}$y&|Z7qdxwsfuIqRdTCB&K{S|bQzvPA6W_TZb z#_P}c%a<>@p_Aoe0lbxdrhk}24{E@J8g{6*f>Z-|rJ2rR0Wl@o)bTVrM?~`{y`>wv z3p}v!r)=r5$)BuVxeh^mhJ&9Ii{o}O%aEGF1V-$OIfo0tNCQ5Ag=<3wPoK6s-Cke6 zHCT84I$&RWjtf1CSc@sJ2{d%Nav^K(O#Xo;)o<5^?!m81=iz>TskIUXeP0GgXnxY(LA_tDAa6(?{q_7dbWP1&nb1u+2P@gH))y9 z;&2+U-z(R1*rXp$7+W1qY-e(~0E9yC5V+{j^_+**|IHw7&7A+a>K-5I>$Q>P;S1X_ z?+A4~_y*E)9GvJX<6je5ts$}wv=^UlCo7uEuL*!T-0DLr#ieOnpBs+`(^O2@d~Pw- zMi#^rO_6}ehse@AMuUW73^W4m=_|j@1mT&#PMC(W6M7F z4VFvT@80+Ib&O>R^$?zIA&-0-%gk-`SV4sT7u)T>#}{)G|7#Ba_qX5ls`x!qk#aIn z5GmyigG~2g(Sk5lmMd~gpj}y@6e|8kO6;eUKsc*2{b#$ePdeS+p4la*$3;TqEg)3| z$Xh^)g~!BkY&Gfe77(%I@Mp7<^zcz^B*(8NIT+PtW;S?OY+JB5#js)hAizJhBu6>-P=rJd!D;9*Q=MS9S-e{S0EoGQ@%o8cA||7IwlbMBRgEjWeU@Ri6_ zhe@cpZn^+;Ij=o4f~`qk?CS{-6}W!lR>ges=~4d!uDAl4fdK@40>M{kF6j4T1nzo0 zi%iKJ4Z6o*^1YAxgTawCU4Hj^U#Fx$SpaKbJ@gQoSzmQSA1_T4l1%{aHCrhCO#i%W zw|gdEHvSRn>)TsHM-&DepBu(=V~-v~m?49O?7TMoVF;c_SVg~uDrvnI1PNoJ0(c%+ z{oKskYb!AC2qTJsjlRQebyo^fJ?bk3v{nt4MYvI7`>4c@x(a-7g$U(jP}Zpv15r|H zBRqAKRoVu&x@u!DsIJ%&qj?Gi$IvZOZ$XuMg$OaP|>d!{i`8Z~N z*?&DZ=qd2`HG(E2j1>XG2+tyqd<%A$!j$R0MHB{znsTWSL_w5gf%L_>q9F_|RTc<& zYkJr;i)Vy)(W3f0Ee-$z^jM0T7cNeMH=0BleEF*tQme+*>J zC{FTZbk|LUb=pt_~5T0^w_-Xo$h7t z56AiUo9_?b|M>gA_wen1sQYGcPP~u*x-kF!@b=>OarZ5G?Y#N$;Jv%L|KmS&svDeO z7}Oibq4lla2v3|PwwRE;26P_jyg-i``UekSGKJn|M$#?(;qjrRez^`LPx-s4P7rMHX?IQs7#|7x_?5MeLpqyKc`ayX-IHa5YOm5{5G6J^Uie; zBezccYaZjg_Pi|mH;{5B221MbhW`FyMIpI?CPep4j+`3$?|@L&3c>pgt)FA;EU^hc zIHT+LBme2T=&*M(be0B@-TFz*nn2D5J%aD+1+)>o#cM1-4EUxKiG%rH@v5@}^tZs|9+(_?69rX3ZrU0FEq#a3ouWUd39A()Po zQV_|&jMf!^iCGpR5TVjjp{BIvQt1<_SS8+yY3s6U87NuNZcwE0Rd9&gDW%1$gs?;V zrJ}Y_7fz90k?ebA`LkqN*z_LZK+3EVeshwx=S&|bWZJ99UXjRQGY?QDMd!WttJu`%rah)!lc%J{J# zx%IE;#|m7g#%q0AuXQ)R+BR={I{NwzH0z+RU@j0#MYdF_TMg3P_E@?503oQx#+C+%%y}HyGYP zKw>AT18rG0m>2;Slb8PrrEI~U*wyIf)vp*~3qC+EWSw){W-C}F|5pM@m;{`k2Oxn| zOD2FE%crh5*__nB>i6@iGo+U43Q$PORza1*5$C3>=Aa5e=H#7L&N`LhzLQLJSpsIW zTmJ)h>z^s-9(Brc?psRTy119YlEt?^-G*sz1K7z%Q**ycnrKfYCHq8c6gW^dk!&1VeFPha zxR<|QxNM>TT)*yxk9*J0V@Dk2e}Hs?u1^&=r;uWBm~=$REZEQQC-3{1FSY; z-MMHLq5d2WM>TYvrt7XGi72t2AMt$W zuX@Q=1g-|66@lFa$x6^~2=++`_DFJ2l`n(8(T6O4ajcEX9#R#nxy7_kmi1n+8BBuIlsz@dJJjZfx;q?vYj4(Lr0n~p#*!|#0k{_?EIA3kJ}sg z#S$})jwPomC8V&WiIxwrUe+kK)3#=e6!HyQ>#5 z(QjB|Z&>3rZ`(CP2{kGVl}KxKw%Dn<5TV|%#J;e^pq;m=8>55TBD94tYhsNuf!gZq zAss@)B74IkoxH8&7%kKk3od|I6MLwKgsU@$1pS6J_J%dOd7IW_Nb86?i@?b9s-tTS z3OPYB_*|a+Mc6(;(sUDGe!cC;stNn^6xha%;a-&m$z{q2VP6k zQwrOiY>XF-S8#t%QG2BX7INoTSwJQXstklKN&MM6gfpcQ-hqegGlY{>5k8;=Bor*% zO(lc}7cr#5ItcLVdnZWiFI&N&WLk|s)Dk^HR8^8vle3}JBQd9nJ*s7gBeS>$K{g@W zUm2y0+|4ShG*3}jImATy-#=NUh5gFZ8XQZ`U_??g+ApfECgLC@`oasF1QJnINyI_d zt1VJU??!D1hMmO^l zH3_&LS2crd=%zMzbnb1jy=RrM-?*|@gHy@TAD>095TWr{QpwFJ2@u&C@OY!)9AFG{ z^A#k}a}R_IUQz0BxO_xAIvp-RSP;k;w^9n~$D(v+Jzn;TE(yPA^8%21s|$|E2qTIB zb-a*IMq~^zukq!7|I1mLb11mF^Bp7v%wYD1Kj|EL*O2$AcoAzj1)P7GsWzry=9^mK zr%#Xidw)q%ufG(CuuWk<{~kT&4$5yT{pec7jjo)TkHwWU%f3ee#QJu9yH4%ClrTAC zv{0^?SYpkEH|IqZAo?|0c0qxy*vgA8n~T=4#4tCVqQYlo%7Mj~yR4C%ASA(fdjoKne}y~X;scXEW zy2hI_wnP7Z-Fjy9+_HXOVvBH~{)O)0rLmhQ;r2&GyDVCeiv5|?he7a-5kPPse#>^~ zk}Mc*hf_iqZl~Mh7A6H2pdi21*al>tTw>v!@lt!0&)N+n(Bn7v@&&(vQ@)siCJ|SO zuD2?6c;YBpyf}w+`-Xl4zVXvYu`<)=_W;>oV#D7ZFEilF#C8AN6haQpw`x-^%ZSrw z%wDXz@HddSj&D>}1;O2+=+0vQ9D;s!8r}3fJip17wX14fCzz?Mg)LbX`Az*olk6ug z(Ve8T@;0|mvnp1b!X@A-)-H=poU#Nfl;;X1WtXTyxK%1H$}R2Jlt6Dvpg;Q(=ou5I zWoJrOu#*+V8}kCcdx zr7-;r*<0App}4zzem)nU4qkP9C&3_JUWB)FZG2W=E3xmVhW_VtN+1oxMOP5d=-mId zzKsQx+ic(_ME6_mKRFcGV{S0Bl?mJk4mFGrNW%%krRQbn;tI}yxw`fo>X?ol-uYD3 zJ55l^Ggpj)2~l$Hl2cyVb5k5?dR@xycqIi@N$c_H4~=c-C4~Z@Oc=uF>}s22Wr6{r zJWy#HN)jcrH4;J9!XfD$xT;Ga_OZ4(1GtfrqP^OfmMpPiT%lG9I&Hg07->=^2V*1N zRhrj)T5`YKg03X7-ZyZsA!aLI*&&Gc7rO#Ba8WDPtmy8cEZiKcUj)2@nV%ZPPId-G zsAgtd(n$6G)bPM{voW}v3r}yO(APEekYjPI<(oB$w))wi9sO7G`1)2_`X}9|XyWKi z9R1nH(UmO9q!dt^e1VX8)TfTO0MySJhp#AFz#KDlFS0+5dy>*8zS_Qlgw4&i^$Y}H zo}{3G;Jx*viHHZ(lr(5c8Z;#h>XS|V+)^v!$|ViPvXTZ|0OoHf z6Vw{c0m57h8k*bZkFJ4bK`XKX49z_fJuj461Frg#Ue zUSzXZQ@nXH=s9AU6*)USJKPzAqd=iVb!+ie9&dK^*^J_{P?0ZRrM$}v#A4~45@=K0i0P}9?mXk2? zo-)BwSR)}0WLpW6LP#z+c6UT4PtG(+`agi|Pb3wExNSt<3U^>~Nk84Ypj^m- z_Tr_>yHLl2t)Nrh=2!X!wNw)lV7+tY+SCmBfi**fRi3hhn6-$6ZBz`R`ga)lLF_BX zT=k{b9DToc-KO;eOG3S>Flw<-U`^8#)$$4?#hafj_T9T<^9mVXl z0PZJWPThXG@K9L`UYJnwjzZwkmG<=}EiY=ASm+ zVW-`WJI?)+qvJu_(8E+U{@wSUT|n8yTqj)J*2}6+@~)iO5cg)2CBx@OCwznM{{sL3 N|NlLT6V5^r0RSE$l(9nI~qq-D?lcmWlAp zHBTN)il_t46Z6b<2{o=RN5<*p*ld}1&?m^nyvJd?|I)X*=DKArAf3B5F;6}{m>4Wr z&)<;)Ef?YTsYP9X0{P$QJ2YFS4T{`VB0<eQrDhQ`Fw{YZ=h@UH`f9@EL!mXl9W17*ptD z;H2?0e*4Y9$Pq0+QyCl<%+W|ko}g$az<7i$g|fRP5g2Mf=E44ygbbbI}Q zdEH{&`Rjml>AN2EDPk>Vz#*_@(v=5Ub9ed=w5WN#Zkay(x;= z!h5Z+StGJ^D0^vucX9=?V@Ut37(z(hc)6HDKLh4He{3LP03&J=8+L!tKR7<@AN7Yv z*PFowE%gQ`dB$ELsO8+X2tzZ8)f+_T%cw-&v)6RznU_(dNp<2>8k}G6#B1#hhBNUd%ya>8;ILC?*Cl$4!h=h-OBRt znd4fwgt|U_18F%9PIOiAuZ8UP2-(k#XP>U8E1Ju%34l4=Dj=2O(loA5jYorNDkf|` zx0o7Z8)AxPNJ8WzWE(!CLBcTx27%7(rD!un_(rS~rlD+wwt*sOZDULsf6QEe0c~Tt zGJx@?Pz@~u7$et)mhtS{99eS%VH+_zXYW`n9f~aW>q`$!no+U0kCZV;~i@? z2V~w(^4){^ip?9Ct$Ne%z=ELvjo9mAml%nJ#8PkOusL@qhRY*vyU0>wsh{A!YeQC((cgNMbo1$$Er8`cj3{8LMU zerKSN1g!HpxZ&(_4hi+$)npEQyQEZ5fl)~avBb}XWRMC56-2p^C?OqkP~Ewt2q;_2 z!WC_eJK!u~W$RYP9q*S;mOFxnF_{(VLC5*IX}^1*LHln;TTuL)k%Z29Z-{Q;0KSFbTecdtAvKDK6E-K{LkO4iGv{Kw8Ok} zmjH*oH}RSIKcn$E`&@^i-4YOaD$+@})9;8fLXndg84^CX*dwqCc)G5Pmw<{%6=BZUAjJ<%sV_)7?VHt%M) zci#WQb^m(v{r>wOfB*LmzWopN-V9HP|M6dE)?fE;&wiiu-jdhun-BN?yNkO&{zGSG zixUikdgZ#bzSSG!sk_8B6S9|p&Lf={=&_dh!3UVkpud@sbPF*&KD5*?mw^PzPF>3s z6f@|txt{}s+3*;Qg8`$V%jMKTWX=hdX|$FpGL+f($!-LTQ~kSk8xi6eir>JNI4UOC3Ve~`TlH0A-RGUM0ZS%9JkEh0imoFf_EEQKgHNx zVhet7$CvNN;_135**h6JOM}R6{iJ41B4>jh!S{7?V#>8Ym+K7LcA`JZ1YizkT8vctZDFU(>#~BbEQF1`)uEO^c9(rgTp#6zX)pr&ZS+lP6eYT(ER481)4^7* z+RDB)sfly;kE1>^!cD3#N!UJSnCME0v@$EV^c&mhA+$8pj*8r_ES&dZD>E=M*MZKE zOh-y7$Yfwf>q@}HEDITkQ0eJVQ(ANB^a)k07H`G0b=kEHl&owwDAM>kIK=Ig(&AM? z*rEMWQCsK>r%10T_Pw(FS@Kxe({%L*9aV;M=mMXun4X7rs61sG40_(b1X6MS@|NKM z$vf+f5XcABWIX?B8P84Nq6u6yfr}<^(F87Mw{-T0D^;3{6hg?k4-Vgi8!?UC!P!nK;*B9A#sq3cx1?`nLL z#y4qvlb^^pslkuHp~`4I(%zD(rF+GPcxtW{$sm);am8imSvHs$0hN=Nc!kz!!JpXG=;hU~7-JjWLqBAlQ^#Q|Smpe$0+RF)aQ-;} z1*BR|0;sWk?2403rvAmCpI4pX9k_)Ro#3$RWBI`0`|5LnVd_a&t*J=8)&O==udpeOo+jF%qV}F_jRFU%CX$UqtB+vg zPowhY&q>yjeVsF^uK#>KIA%uE@p9K)>VhrsV=oXwI zN55f>yv4?(0xH@wv&~I2{Usz+<$=kG^Kw3xCSq4VM1e+jn zF(+sSpUacK4BMU4d=p@P$^qs^R$L6bibQ~Sz`uob1h579Ij89OPAO5Lf})ZXHlTn8 zZQ5bu;H-iNq2K18`$cM1_1bM^NzQN{?IeO?ii)e5mpNGw6H5JS$w7myPYwi&sdsds zQsw47J7Vyw$)%nm&qJ`_8%R&dZ53H0ddj!RUl3suFL=wgxSSv`lTXQQ-o|+xcrAyX zQrPZfW4vIzlH&VC?X?cDkXun@0a-ApG7!2X@n`Q4&Xh`c2kx`a5KdM__<$CWP_Xbe zl@RVd#E=T>Ai%Hh-QZY%*$M_FkJX5wmgo_ps&gncIU8C%5_78AquS|kWEIyS$Q}q6 zE2DKHceBbm%~KRs4l&vJ@1LyF!hW698V(dW7?ISB_KT{ki8u&_zVO2)fkf0*5^>P= zYKs)oyHOjGVP|!PypfKA2jg2UWF^)+K3J-fKkg?l5gxNvW>@6iCUzFuFiQ@bxEOwJfB zlq)8dSaa#MHxUKMevOu0QeZ2#^0J#^qBSfr%uT1L@L8F1VDaTHYa}NKNirTm0o4%W zL?eNjpquT(4(SqZf@2>0z0!b4Al1rT8dUa@Az$|vrAWqUm8{vT^C`naMLOk1ua2?h zGWA5OWWy-Is7#Jz(3l4lS$jRRBlV74!$a|+ipEEFbUQLX6lhCK5Oqr}Go-0&yra6t zn=-Z|aXDT+GkR*pH0enxWO~U+GVkcQ$?2RyD4!i)9UxAk=_pxj{t=U3=& zv;X8$;7qu|%vL6FBf^XMMZ&2noG@DYeugeC;0>@=m%d9~%XPv#pQ?JN2}*h5icvBl zTFzZ^%1e80nj=lGOW7T-q@XHkJwATY*mho0C;-ZYAp~bv+axO!3<%|cPTNqDXql~% z2&xtiN$`m}wM`D-MoP-|YGYcm#ENl+S}EwX?H*yINwpk|jd<5-UURnOUbzK_ z9mTk?#Ge zVZwE@F}RxxPp_lU*Cq6kYjdpCn>C5H=E<-V{Z}%5eJd^flkQVAar7pR{^aB6S{7we z3TREfM94gtGuK}LD$b00}Rf46kMnW9O zwh|#w>2RQ>a3ZL3mD8!DaN?*oD(6>IIfD8!a~nP0O>AmVJ-qfb1CMr0)ryrFg{Lu_nWNZ&PcI z6m^A!XuUv$j*!hbrDR@kw*M9v*k^&&etEqV-{#`_Jy2a4HM3ud{l|*)z8iuTpQ^I2 z_*4t?icdRa-3Zr)aT~^c;*5LLRh-DVT4IV8aI&0|72)e!igPpFbI#-N;x?O>d0d=$eG6#3%9hagvQR+6b=9p&t_0Ny7dr>;L;^Cg}@ z;6VEd&hoe_1~-SFa)6g&tK=wlq9b!q&CGayTOGIj_^8wA@J}1>u-oax9p}N(!Qrse zGQ*>4{JZd;JwVySTqa!I_KT`c@~)iO5cg)2CBx_a9^at*{{R30|NrD3;iCTt0RSL< Bv!eh2 diff --git a/cmd/boostd/index.go b/cmd/boostd/index.go index f4947d005..1fbeee504 100644 --- a/cmd/boostd/index.go +++ b/cmd/boostd/index.go @@ -1,6 +1,7 @@ package main import ( + "fmt" bcli "github.com/filecoin-project/boost/cli" lcli "github.com/filecoin-project/lotus/cli" "github.com/urfave/cli/v2" @@ -11,6 +12,8 @@ var indexProvCmd = &cli.Command{ Usage: "Manage the index provider on Boost", Subcommands: []*cli.Command{ indexProvAnnounceAllCmd, + indexProvAnnounceLatest, + indexProvAnnounceLatestHttp, }, } @@ -31,3 +34,54 @@ var indexProvAnnounceAllCmd = &cli.Command{ return napi.BoostIndexerAnnounceAllDeals(ctx) }, } + +var indexProvAnnounceLatest = &cli.Command{ + Name: "announce-latest", + Usage: "Re-publish the latest existing advertisement to pubsub", + Action: func(cctx *cli.Context) error { + ctx := lcli.ReqContext(cctx) + + napi, closer, err := bcli.GetBoostAPI(cctx) + if err != nil { + return err + } + defer closer() + + c, err := napi.BoostIndexerAnnounceLatest(ctx) + if err != nil { + return err + } + + fmt.Printf("Announced advertisement with cid %s\n", c) + return nil + }, +} + +var indexProvAnnounceLatestHttp = &cli.Command{ + Name: "announce-latest-http", + Usage: "Re-publish the latest existing advertisement to specific indexers over http", + Flags: []cli.Flag{ + &cli.StringSliceFlag{ + Name: "announce-url", + Usage: "The url(s) to announce to. If not specified, announces to the http urls in config", + Required: false, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := lcli.ReqContext(cctx) + + napi, closer, err := bcli.GetBoostAPI(cctx) + if err != nil { + return err + } + defer closer() + + c, err := napi.BoostIndexerAnnounceLatestHttp(ctx, cctx.StringSlice("announce-url")) + if err != nil { + return err + } + + fmt.Printf("Announced advertisement to indexers over http with cid %s\n", c) + return nil + }, +} diff --git a/cmd/boostd/init.go b/cmd/boostd/init.go index 76ffc7838..97ed8fa2a 100644 --- a/cmd/boostd/init.go +++ b/cmd/boostd/init.go @@ -493,7 +493,13 @@ func migrateMarketsConfig(cctx *cli.Context, mktsRepo lotus_repo.LockedRepo, boo // Clear the DAG store root dir config, because the DAG store is no longer configurable in Boost // (it is always at /dagstore rcfg.DAGStore.RootDir = "" - rcfg.IndexProvider = mktsCfg.IndexProvider + rcfg.IndexProvider = config.IndexProviderConfig{ + Enable: mktsCfg.IndexProvider.Enable, + EntriesCacheCapacity: mktsCfg.IndexProvider.EntriesCacheCapacity, + EntriesChunkSize: mktsCfg.IndexProvider.EntriesChunkSize, + TopicName: mktsCfg.IndexProvider.TopicName, + PurgeCacheOnStart: mktsCfg.IndexProvider.PurgeCacheOnStart, + } rcfg.IndexProvider.Enable = true // Enable index provider in Boost by default if fromMonolith { diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md index d223d9b25..5e9a4991f 100644 --- a/documentation/en/api-v1-methods.md +++ b/documentation/en/api-v1-methods.md @@ -23,6 +23,8 @@ * [BoostDealBySignedProposalCid](#boostdealbysignedproposalcid) * [BoostDummyDeal](#boostdummydeal) * [BoostIndexerAnnounceAllDeals](#boostindexerannouncealldeals) + * [BoostIndexerAnnounceLatest](#boostindexerannouncelatest) + * [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp) * [BoostMakeDeal](#boostmakedeal) * [BoostOfflineDealWithData](#boostofflinedealwithdata) * [Deals](#deals) @@ -563,6 +565,41 @@ Inputs: `null` Response: `{}` +### BoostIndexerAnnounceLatest + + +Perms: admin + +Inputs: `null` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + +### BoostIndexerAnnounceLatestHttp + + +Perms: admin + +Inputs: +```json +[ + [ + "string value" + ] +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + ### BoostMakeDeal diff --git a/go.mod b/go.mod index 11792ad57..c9c1b7166 100644 --- a/go.mod +++ b/go.mod @@ -76,7 +76,7 @@ require ( github.com/ipld/go-car/v2 v2.7.0 github.com/ipld/go-ipld-prime v0.20.0 github.com/ipld/go-ipld-selector-text-lite v0.0.1 - github.com/ipni/index-provider v0.11.1 + github.com/ipni/index-provider v0.11.2 github.com/ipni/storetheindex v0.5.10 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jpillora/backoff v1.0.0 diff --git a/go.sum b/go.sum index 81e692d94..e7cf19572 100644 --- a/go.sum +++ b/go.sum @@ -951,8 +951,8 @@ github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5dIFn6iw4L/Hh+kS1Y= github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM= -github.com/ipni/index-provider v0.11.1 h1:viNfSBvZA9G+Qe6/FGqfZtavnu4tTSfGUoWEECavqoI= -github.com/ipni/index-provider v0.11.1/go.mod h1:gB/wN4Mdz4MzikQubjyRRV97iS5BkD4FKB0U/bF/dY4= +github.com/ipni/index-provider v0.11.2 h1:nvykWK+/ncPTqHiuiJdXp/O0UF0V7iWesjHGKX//NYc= +github.com/ipni/index-provider v0.11.2/go.mod h1:gB/wN4Mdz4MzikQubjyRRV97iS5BkD4FKB0U/bF/dY4= github.com/ipni/storetheindex v0.5.10 h1:r97jIZsXPuwQvePJQuStu2a/kn+Zn8X4MAdA0rU2Pu4= github.com/ipni/storetheindex v0.5.10/go.mod h1:SJKFCnSx4X/4ekQuZvq8pVU/7tmxkEv632Qmgu3m2bQ= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index e1840c5fa..fc25cb2c9 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math" + "net/url" "os" "path/filepath" @@ -22,6 +23,7 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" provider "github.com/ipni/index-provider" + "github.com/ipni/index-provider/engine" "github.com/ipni/index-provider/engine/xproviders" "github.com/ipni/index-provider/metadata" "github.com/libp2p/go-libp2p/core/crypto" @@ -262,6 +264,35 @@ func (w *Wrapper) IndexerAnnounceAllDeals(ctx context.Context) error { return merr } +func (w *Wrapper) IndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) { + e, ok := w.prov.(*engine.Engine) + if !ok { + return cid.Undef, fmt.Errorf("index provider is disabled") + } + return e.PublishLatest(ctx) +} + +func (w *Wrapper) IndexerAnnounceLatestHttp(ctx context.Context, announceUrls []string) (cid.Cid, error) { + e, ok := w.prov.(*engine.Engine) + if !ok { + return cid.Undef, fmt.Errorf("index provider is disabled") + } + + if len(announceUrls) == 0 { + announceUrls = w.cfg.IndexProvider.Announce.DirectAnnounceURLs + } + + urls := make([]*url.URL, 0, len(announceUrls)) + for _, us := range announceUrls { + u, err := url.Parse(us) + if err != nil { + return cid.Undef, fmt.Errorf("parsing url %s: %w", us, err) + } + urls = append(urls, u) + } + return e.PublishLatestHTTP(ctx, urls...) +} + func (w *Wrapper) Start(ctx context.Context) { // re-init dagstore shards for Boost deals if needed if _, err := w.DagstoreReinitBoostDeals(ctx); err != nil { diff --git a/node/config/def.go b/node/config/def.go index 18364c996..b8a216a62 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -171,7 +171,7 @@ func DefaultBoost() *Boost { MaxConcurrencyStorageCalls: 100, GCInterval: lotus_config.Duration(1 * time.Minute), }, - IndexProvider: lotus_config.IndexProviderConfig{ + IndexProvider: IndexProviderConfig{ Enable: true, EntriesCacheCapacity: 1024, EntriesChunkSize: 16384, @@ -179,6 +179,17 @@ func DefaultBoost() *Boost { // format: "/indexer/ingest/" TopicName: "", PurgeCacheOnStart: false, + + Announce: IndexProviderAnnounceConfig{ + AnnounceOverHttp: false, + DirectAnnounceURLs: []string{"https://cid.contact/ingest/announce"}, + }, + + HttpPublisher: IndexProviderHttpPublisherConfig{ + Enabled: false, + PublicHostname: "", + Port: 3104, + }, }, } return cfg diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index fa3623b9e..314596306 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -93,7 +93,7 @@ your node if metadata log is disabled`, }, { Name: "IndexProvider", - Type: "lotus_config.IndexProviderConfig", + Type: "IndexProviderConfig", Comment: ``, }, @@ -436,6 +436,102 @@ for any other deal.`, Comment: `The port that the graphql server listens on`, }, }, + "IndexProviderAnnounceConfig": []DocField{ + { + Name: "AnnounceOverHttp", + Type: "bool", + + Comment: `Make a direct announcement to a list of indexing nodes over http. +Note that announcements are already made over pubsub regardless +of this setting.`, + }, + { + Name: "DirectAnnounceURLs", + Type: "[]string", + + Comment: `The list of URLs of indexing nodes to announce to.`, + }, + }, + "IndexProviderConfig": []DocField{ + { + Name: "Enable", + Type: "bool", + + Comment: `Enable set whether to enable indexing announcement to the network and expose endpoints that +allow indexer nodes to process announcements. Enabled by default.`, + }, + { + Name: "EntriesCacheCapacity", + Type: "int", + + Comment: `EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement +entries. Defaults to 1024 if not specified. The cache is evicted using LRU policy. The +maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize and +the length of multihashes being advertised. For example, advertising 128-bit long multihashes +with the default EntriesCacheCapacity, and EntriesChunkSize means the cache size can grow to +256MiB when full.`, + }, + { + Name: "EntriesChunkSize", + Type: "int", + + Comment: `EntriesChunkSize sets the maximum number of multihashes to include in a single entries chunk. +Defaults to 16384 if not specified. Note that chunks are chained together for indexing +advertisements that include more multihashes than the configured EntriesChunkSize.`, + }, + { + Name: "TopicName", + Type: "string", + + Comment: `TopicName sets the topic name on which the changes to the advertised content are announced. +If not explicitly specified, the topic name is automatically inferred from the network name +in following format: '/indexer/ingest/' +Defaults to empty, which implies the topic name is inferred from network name.`, + }, + { + Name: "PurgeCacheOnStart", + Type: "bool", + + Comment: `PurgeCacheOnStart sets whether to clear any cached entries chunks when the provider engine +starts. By default, the cache is rehydrated from previously cached entries stored in +datastore if any is present.`, + }, + { + Name: "Announce", + Type: "IndexProviderAnnounceConfig", + + Comment: ``, + }, + { + Name: "HttpPublisher", + Type: "IndexProviderHttpPublisherConfig", + + Comment: ``, + }, + }, + "IndexProviderHttpPublisherConfig": []DocField{ + { + Name: "Enabled", + Type: "bool", + + Comment: `If not enabled, requests are served over graphsync instead.`, + }, + { + Name: "PublicHostname", + Type: "string", + + Comment: `Set the public hostname / IP for the index provider listener. +eg "82.129.73.111" +This is usually the same as the for the boost node.`, + }, + { + Name: "Port", + Type: "int", + + Comment: `Set the port on which to listen for index provider requests over HTTP. +Note that this port must be open on the firewall.`, + }, + }, "LotusDealmakingConfig": []DocField{ { Name: "PieceCidBlocklist", diff --git a/node/config/types.go b/node/config/types.go index 2aae18cb8..d8c1c5b89 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -49,7 +49,7 @@ type Boost struct { LotusDealmaking lotus_config.DealmakingConfig LotusFees FeeConfig DAGStore lotus_config.DAGStoreConfig - IndexProvider lotus_config.IndexProviderConfig + IndexProvider IndexProviderConfig } func (b *Boost) GetDealmakingConfig() lotus_config.DealmakingConfig { @@ -279,6 +279,62 @@ type ContractDealsConfig struct { From string } +type IndexProviderConfig struct { + // Enable set whether to enable indexing announcement to the network and expose endpoints that + // allow indexer nodes to process announcements. Enabled by default. + Enable bool + + // EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement + // entries. Defaults to 1024 if not specified. The cache is evicted using LRU policy. The + // maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize and + // the length of multihashes being advertised. For example, advertising 128-bit long multihashes + // with the default EntriesCacheCapacity, and EntriesChunkSize means the cache size can grow to + // 256MiB when full. + EntriesCacheCapacity int + + // EntriesChunkSize sets the maximum number of multihashes to include in a single entries chunk. + // Defaults to 16384 if not specified. Note that chunks are chained together for indexing + // advertisements that include more multihashes than the configured EntriesChunkSize. + EntriesChunkSize int + + // TopicName sets the topic name on which the changes to the advertised content are announced. + // If not explicitly specified, the topic name is automatically inferred from the network name + // in following format: '/indexer/ingest/' + // Defaults to empty, which implies the topic name is inferred from network name. + TopicName string + + // PurgeCacheOnStart sets whether to clear any cached entries chunks when the provider engine + // starts. By default, the cache is rehydrated from previously cached entries stored in + // datastore if any is present. + PurgeCacheOnStart bool + + Announce IndexProviderAnnounceConfig + + HttpPublisher IndexProviderHttpPublisherConfig +} + +type IndexProviderAnnounceConfig struct { + // Make a direct announcement to a list of indexing nodes over http. + // Note that announcements are already made over pubsub regardless + // of this setting. + AnnounceOverHttp bool + + // The list of URLs of indexing nodes to announce to. + DirectAnnounceURLs []string +} + +type IndexProviderHttpPublisherConfig struct { + // If not enabled, requests are served over graphsync instead. + Enabled bool + // Set the public hostname / IP for the index provider listener. + // eg "82.129.73.111" + // This is usually the same as the for the boost node. + PublicHostname string + // Set the port on which to listen for index provider requests over HTTP. + // Note that this port must be open on the firewall. + Port int +} + type FeeConfig struct { // The maximum fee to pay when sending the PublishStorageDeals message MaxPublishDealsFee types.FIL diff --git a/node/impl/boost.go b/node/impl/boost.go index 5d2927966..736cb6c4c 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -145,6 +145,14 @@ func (sm *BoostAPI) BoostIndexerAnnounceAllDeals(ctx context.Context) error { return sm.IndexProvider.IndexerAnnounceAllDeals(ctx) } +func (sm *BoostAPI) BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) { + return sm.IndexProvider.IndexerAnnounceLatest(ctx) +} + +func (sm *BoostAPI) BoostIndexerAnnounceLatestHttp(ctx context.Context, announceUrls []string) (cid.Cid, error) { + return sm.IndexProvider.IndexerAnnounceLatestHttp(ctx, announceUrls) +} + func (sm *BoostAPI) BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string, delAfterImport bool) (*api.ProviderDealRejectionInfo, error) { res, err := sm.StorageProvider.ImportOfflineDealData(ctx, dealUuid, filePath, delAfterImport) return res, err diff --git a/node/modules/storageminer_idxprov.go b/node/modules/storageminer_idxprov.go index 9e3585672..0b41dc600 100644 --- a/node/modules/storageminer_idxprov.go +++ b/node/modules/storageminer_idxprov.go @@ -5,13 +5,14 @@ import ( "fmt" "github.com/filecoin-project/boost/build" "github.com/filecoin-project/boost/indexprovider" + "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/boost/node/modules/dtypes" "github.com/filecoin-project/boost/retrievalmarket/types" + "github.com/filecoin-project/boost/util" "github.com/filecoin-project/go-address" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-data-transfer/transport/graphsync" datatransferv2 "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/lotus/node/config" lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -77,7 +78,8 @@ func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHo "pid", marketHost.ID(), "topic", topicName, "retAddrs", marketHost.Addrs()) - // If announcements to the network are enabled, then set options for datatransfer publisher. + + // If announcements to the network are enabled, then set options for the publisher. var e *engine.Engine if cfg.Enable { // Join the indexer topic using the market's pubsub instance. Otherwise, the provider @@ -93,14 +95,36 @@ func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHo // The extra data is required by the lotus-specific index-provider gossip message validators. ma := address.Address(maddr) opts = append(opts, - engine.WithPublisherKind(engine.DataTransferPublisher), - engine.WithDataTransfer(dtV1ToIndexerDT(dt, func() ipld.LinkSystem { - return *e.LinkSystem() - })), - engine.WithExtraGossipData(ma.Bytes()), engine.WithTopic(t), + engine.WithExtraGossipData(ma.Bytes()), ) - llog = llog.With("extraGossipData", ma, "publisher", "data-transfer") + if cfg.Announce.AnnounceOverHttp { + opts = append(opts, engine.WithDirectAnnounce(cfg.Announce.DirectAnnounceURLs...)) + } + + // Advertisements can be served over the data transfer protocol + // (on graphsync) or over HTTP + if cfg.HttpPublisher.Enabled { + announceAddr, err := util.ToHttpMultiaddr(cfg.HttpPublisher.PublicHostname, cfg.HttpPublisher.Port) + if err != nil { + return nil, fmt.Errorf("parsing HTTP Publisher hostname '%s' / port %d: %w", + cfg.HttpPublisher.PublicHostname, cfg.HttpPublisher.Port, err) + } + opts = append(opts, + engine.WithPublisherKind(engine.HttpPublisher), + engine.WithHttpPublisherListenAddr(fmt.Sprintf("0.0.0.0:%d", cfg.HttpPublisher.Port)), + engine.WithHttpPublisherAnnounceAddr(announceAddr.String()), + ) + llog = llog.With("publisher", "http", "announceAddr", announceAddr) + } else { + opts = append(opts, + engine.WithPublisherKind(engine.DataTransferPublisher), + engine.WithDataTransfer(dtV1ToIndexerDT(dt, func() ipld.LinkSystem { + return *e.LinkSystem() + })), + ) + llog = llog.With("extraGossipData", ma, "publisher", "data-transfer") + } } else { opts = append(opts, engine.WithPublisherKind(engine.NoPublisher)) llog = llog.With("publisher", "none") diff --git a/util/addr.go b/util/addr.go new file mode 100644 index 000000000..f171cffc9 --- /dev/null +++ b/util/addr.go @@ -0,0 +1,26 @@ +package util + +import ( + "fmt" + ma "github.com/multiformats/go-multiaddr" + "net" + "strings" +) + +func ToHttpMultiaddr(hostname string, port int) (ma.Multiaddr, error) { + if hostname == "" { + return nil, fmt.Errorf("hostname is empty") + } + + var saddr string + if n := net.ParseIP(hostname); n != nil { + ipVersion := "ip4" + if strings.Contains(hostname, ":") { + ipVersion = "ip6" + } + saddr = fmt.Sprintf("/%s/%s/tcp/%d/http", ipVersion, hostname, port) + } else { + saddr = fmt.Sprintf("/dns/%s/tcp/%d/http", hostname, port) + } + return ma.NewMultiaddr(saddr) +} diff --git a/util/addr_test.go b/util/addr_test.go new file mode 100644 index 000000000..810020787 --- /dev/null +++ b/util/addr_test.go @@ -0,0 +1,44 @@ +package util + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestToHttpMultiaddr(t *testing.T) { + tcs := []struct { + hostname string + port int + expected string + expectErr bool + }{{ + hostname: "192.168.1.1", + port: 1234, + expected: "/ip4/192.168.1.1/tcp/1234/http", + }, { + hostname: "2001:db8::68", + port: 1234, + expected: "/ip6/2001:db8::68/tcp/1234/http", + }, { + hostname: "example.com", + port: 1234, + expected: "/dns/example.com/tcp/1234/http", + }, { + hostname: "", + port: 1234, + expected: "", + expectErr: true, + }} + + for _, tc := range tcs { + t.Run("", func(t *testing.T) { + ma, err := ToHttpMultiaddr(tc.hostname, tc.port) + if tc.expectErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.expected, ma.String()) + } + }) + } +} From 0de116161b1d4a51a8cc029e6457fdc63261218f Mon Sep 17 00:00:00 2001 From: dirkmc Date: Tue, 23 May 2023 10:57:40 +0200 Subject: [PATCH 04/15] Signal to index provider to skip announcements (#1457) * fix: signal to index provider to skip announcements * fix: ensure multihash lister skip error is of type ipld.ErrNotExists --------- Co-authored-by: LexLuthr --- indexprovider/wrapper.go | 133 ++++++++++++++++++++++++++++----------- 1 file changed, 96 insertions(+), 37 deletions(-) diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index fc25cb2c9..14e7e0c05 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -2,13 +2,19 @@ package indexprovider import ( "context" + "database/sql" "errors" "fmt" + "io/fs" "math" "net/url" "os" "path/filepath" + "github.com/filecoin-project/dagstore/index" + "github.com/ipfs/go-datastore" + "github.com/ipld/go-ipld-prime" + gfm_storagemarket "github.com/filecoin-project/boost-gfm/storagemarket" "github.com/filecoin-project/boost/db" "github.com/filecoin-project/boost/markets/idxprov" @@ -27,7 +33,7 @@ import ( "github.com/ipni/index-provider/engine/xproviders" "github.com/ipni/index-provider/metadata" "github.com/libp2p/go-libp2p/core/crypto" - host "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "go.uber.org/fx" ) @@ -264,6 +270,16 @@ func (w *Wrapper) IndexerAnnounceAllDeals(ctx context.Context) error { return merr } +// While ingesting cids for each piece, if there is an error the indexer +// checks if the error contains the string "content not found": +// - if so, the indexer skips the piece and continues ingestion +// - if not, the indexer pauses ingestion +var ErrStringSkipAdIngest = "content not found" + +func skipError(err error) error { + return fmt.Errorf("%s: %s: %w", ErrStringSkipAdIngest, err.Error(), ipld.ErrNotExists{}) +} + func (w *Wrapper) IndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) { e, ok := w.prov.(*engine.Engine) if !ok { @@ -299,42 +315,7 @@ func (w *Wrapper) Start(ctx context.Context) { log.Errorw("failed to migrate dagstore indices for Boost deals", "err", err) } - w.prov.RegisterMultihashLister(func(ctx context.Context, pid peer.ID, contextID []byte) (provider.MultihashIterator, error) { - provideF := func(pieceCid cid.Cid) (provider.MultihashIterator, error) { - ii, err := w.dagStore.GetIterableIndexForPiece(pieceCid) - if err != nil { - return nil, fmt.Errorf("failed to get iterable index: %w", err) - } - - mhi, err := provider.CarMultihashIterator(ii) - if err != nil { - return nil, fmt.Errorf("failed to get mhiterator: %w", err) - } - return mhi, nil - } - - // convert context ID to proposal Cid - proposalCid, err := cid.Cast(contextID) - if err != nil { - return nil, fmt.Errorf("failed to cast context ID to a cid") - } - - // go from proposal cid -> piece cid by looking up deal in boost and if we can't find it there -> then markets - // check Boost deals DB - pds, boostErr := w.dealsDB.BySignedProposalCID(ctx, proposalCid) - if boostErr == nil { - pieceCid := pds.ClientDealProposal.Proposal.PieceCID - return provideF(pieceCid) - } - - // check in legacy markets - md, legacyErr := w.legacyProv.GetLocalDeal(proposalCid) - if legacyErr == nil { - return provideF(md.Proposal.PieceCID) - } - - return nil, fmt.Errorf("failed to look up deal in Boost, err=%s and Legacy Markets, err=%s", boostErr, legacyErr) - }) + w.prov.RegisterMultihashLister(w.MultihashLister) runCtx, runCancel := context.WithCancel(context.Background()) w.stop = runCancel @@ -352,6 +333,84 @@ func (w *Wrapper) Start(ctx context.Context) { }() } +func (w *Wrapper) MultihashLister(ctx context.Context, prov peer.ID, contextID []byte) (provider.MultihashIterator, error) { + provideF := func(proposalCid cid.Cid, pieceCid cid.Cid) (provider.MultihashIterator, error) { + ii, err := w.dagStore.GetIterableIndexForPiece(pieceCid) + if err != nil { + e := fmt.Errorf("failed to get iterable index: %w", err) + if errors.Is(err, index.ErrNotFound) || errors.Is(err, fs.ErrNotExist) { + // If it's a not found error, skip over this piece and continue ingesting + log.Infow("skipping ingestion: piece not found", "piece", pieceCid, "propCid", proposalCid, "err", e) + return nil, skipError(e) + } + + // Some other error, pause ingestion + log.Infow("pausing ingestion: error getting piece", "piece", pieceCid, "propCid", proposalCid, "err", e) + return nil, e + } + + mhi, err := provider.CarMultihashIterator(ii) + if err != nil { + // Bad index, skip over this piece and continue ingesting + err = fmt.Errorf("failed to get mhiterator: %w", err) + log.Infow("skipping ingestion", "piece", pieceCid, "propCid", proposalCid, "err", err) + return nil, skipError(err) + } + + log.Debugw("returning piece iterator", "piece", pieceCid, "propCid", proposalCid, "err", err) + return mhi, nil + } + + // convert context ID to proposal Cid + proposalCid, err := cid.Cast(contextID) + if err != nil { + // Bad contextID, skip over this piece and continue ingesting + err = fmt.Errorf("failed to cast context ID to a cid") + log.Infow("skipping ingestion", "proposalCid", proposalCid, "err", err) + return nil, skipError(err) + } + + // Look up deal by proposal cid in the boost database. + // If we can't find it there check legacy markets DB. + pds, boostErr := w.dealsDB.BySignedProposalCID(ctx, proposalCid) + if boostErr == nil { + // Found the deal, get an iterator over the piece + pieceCid := pds.ClientDealProposal.Proposal.PieceCID + return provideF(proposalCid, pieceCid) + } + + // Check if it's a "not found" error + if !errors.Is(boostErr, sql.ErrNoRows) { + // It's not a "not found" error: there was a problem accessing the + // database. Pause ingestion until the user can fix the DB. + e := fmt.Errorf("getting deal with proposal cid %s from boost database: %w", proposalCid, boostErr) + log.Infow("pausing ingestion", "proposalCid", proposalCid, "err", e) + return nil, e + } + + // Deal was not found in boost DB - check in legacy markets + md, legacyErr := w.legacyProv.GetLocalDeal(proposalCid) + if legacyErr == nil { + // Found the deal, get an interator over the piece + return provideF(proposalCid, md.Proposal.PieceCID) + } + + // Check if it's a "not found" error + if !errors.Is(legacyErr, datastore.ErrNotFound) { + // It's not a "not found" error: there was a problem accessing the + // legacy database. Pause ingestion until the user can fix the legacy DB. + e := fmt.Errorf("getting deal with proposal cid %s from Legacy Markets: %w", proposalCid, legacyErr) + log.Infow("pausing ingestion", "proposalCid", proposalCid, "err", e) + return nil, e + } + + // The deal was not found in the boost or legacy database. + // Skip this deal and continue ingestion. + err = fmt.Errorf("deal with proposal cid %s not found", proposalCid) + log.Infow("skipping ingestion", "proposalCid", proposalCid, "err", err) + return nil, skipError(err) +} + func (w *Wrapper) AnnounceBoostDeal(ctx context.Context, deal *types.ProviderDealState) (cid.Cid, error) { // Filter out deals that should not be announced if !deal.AnnounceToIPNI { From fd35f626907ba66ff6e95fa5a0e4dea240f3e3ce Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Tue, 23 May 2023 13:06:27 +0400 Subject: [PATCH 05/15] release v1.7.3-rc2 (#1460) --- build/openrpc/boost.json.gz | Bin 5619 -> 5619 bytes build/version.go | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index 0680625fed6b2d7b49e393813eea898a42a62e38..ccc996f2b2f3f9e91517bf0d09cf9a65e15337ea 100644 GIT binary patch delta 21 ccmeyY{aJfL3*)+tt!bhh2@><;oCFye0A~CLf&c&j delta 21 ccmeyY{aJfL3uE8L)-+KL+b-dAPJ)aK0A?x&l>h($ diff --git a/build/version.go b/build/version.go index 5c2f02cfc..112065b5e 100644 --- a/build/version.go +++ b/build/version.go @@ -2,7 +2,7 @@ package build var CurrentCommit string -const BuildVersion = "1.7.3-rc1" +const BuildVersion = "1.7.3-rc2" func UserVersion() string { return BuildVersion + CurrentCommit From 24a9bb3d6f7b88c7223e9c753550c51c36be467b Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Tue, 23 May 2023 13:11:47 +0400 Subject: [PATCH 06/15] fix: improve stalled retrieval cancellation (#1449) * refactor stalled retrieval cancel * add ctx with timeout * implement suggestions * update err wrapping * fix: set short cancel timeout for unpaid retrievals only --------- Co-authored-by: Dirk McCormick --- node/config/def.go | 2 +- retrievalmarket/rtvllog/retrieval_log.go | 41 ++++++++++++++------- retrievalmarket/server/gsunpaidretrieval.go | 3 +- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/node/config/def.go b/node/config/def.go index b8a216a62..911177af0 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -98,7 +98,7 @@ func DefaultBoost() *Boost { DealProposalLogDuration: Duration(time.Hour * 24), RetrievalLogDuration: Duration(time.Hour * 24), - StalledRetrievalTimeout: Duration(time.Minute * 30), + StalledRetrievalTimeout: Duration(time.Second * 30), RetrievalPricing: &lotus_config.RetrievalPricing{ Strategy: RetrievalPricingDefaultMode, diff --git a/retrievalmarket/rtvllog/retrieval_log.go b/retrievalmarket/rtvllog/retrieval_log.go index c1f27e350..3711bf577 100644 --- a/retrievalmarket/rtvllog/retrieval_log.go +++ b/retrievalmarket/rtvllog/retrieval_log.go @@ -2,6 +2,7 @@ package rtvllog import ( "context" + "errors" "sync" "time" @@ -295,22 +296,36 @@ func (r *RetrievalLog) gcRetrievals(ctx context.Context) { continue } + var wg sync.WaitGroup for _, row := range rows { - chid := datatransfer.ChannelID{Initiator: row.PeerID, Responder: row.LocalPeerID, ID: row.TransferID} - // Try to cancel via unpaid graphsync first - err := r.gsur.CancelTransfer(ctx, row.TransferID, &row.PeerID) - - if err != nil { - // Attempt to terminate legacy, paid retrievals if we didnt cancel a free retrieval - err = r.dataTransfer.CloseDataTransferChannel(ctx, chid) - } - - if err != nil { - log.Debugw("error canceling retrieval", "dealID", row.DealID, "err", err) - } else { - log.Infof("Canceled retrieval %s, older than %s", row.DealID, r.stalledTimeout) + if row.TransferID <= 0 { + continue } + wg.Add(1) + go func(s RetrievalDealState) { + // Don't wait for more than 5 seconds for the cancel + // message to be sent when cancelling an unpaid retrieval + unpaidRtrvCtx, cancel := context.WithTimeout(ctx, time.Second*5) + defer cancel() + defer wg.Done() + + // Try to cancel an unpaid retrieval with the given transfer id first + err := r.gsur.CancelTransfer(unpaidRtrvCtx, s.TransferID, &s.PeerID) + if err != nil && errors.Is(err, server.ErrRetrievalNotFound) { + // Couldn't find an unpaid retrieval with that id, try + // to cancel a legacy, paid retrieval + chid := datatransfer.ChannelID{Initiator: s.PeerID, Responder: s.LocalPeerID, ID: s.TransferID} + err = r.dataTransfer.CloseDataTransferChannel(ctx, chid) + } + + if err != nil { + log.Debugw("error canceling retrieval", "dealID", s.DealID, "err", err) + } else { + log.Infof("Canceled retrieval %s, older than %s", s.DealID, r.stalledTimeout) + } + }(row) } + wg.Wait() } } } diff --git a/retrievalmarket/server/gsunpaidretrieval.go b/retrievalmarket/server/gsunpaidretrieval.go index 46a42ce58..76d591e8f 100644 --- a/retrievalmarket/server/gsunpaidretrieval.go +++ b/retrievalmarket/server/gsunpaidretrieval.go @@ -30,6 +30,7 @@ import ( ) var log = logging.Logger("boostgs") +var ErrRetrievalNotFound = fmt.Errorf("no transfer found") var incomingReqExtensions = []graphsync.ExtensionName{ extension.ExtensionIncomingRequest1_1, @@ -175,7 +176,7 @@ func (g *GraphsyncUnpaidRetrieval) CancelTransfer(ctx context.Context, id datatr if state == nil { g.activeRetrievalsLk.Unlock() - return fmt.Errorf("no transfer with id %d", id) + return fmt.Errorf("failed to cancel with id %d: %w", id, ErrRetrievalNotFound) } rcpt := state.cs.recipient From 4986e2f33dce34664394dff605a91299f825aea1 Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Wed, 24 May 2023 14:12:20 +0400 Subject: [PATCH 07/15] feat: enable listen address for booster-http (#1461) * enable listen address * modify tests --- cmd/booster-http/http_test.go | 6 +++--- cmd/booster-http/mocks/mock_booster_http.go | 2 +- cmd/booster-http/run.go | 11 +++++++++-- cmd/booster-http/server.go | 17 +++++++++-------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cmd/booster-http/http_test.go b/cmd/booster-http/http_test.go index 4a96ba4db..377c079b7 100644 --- a/cmd/booster-http/http_test.go +++ b/cmd/booster-http/http_test.go @@ -23,7 +23,7 @@ const testFile = "test/test_file" func TestNewHttpServer(t *testing.T) { // Create a new mock Http server ctrl := gomock.NewController(t) - httpServer := NewHttpServer("", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil) + httpServer := NewHttpServer("", "0.0.0.0", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil) err := httpServer.Start(context.Background()) require.NoError(t, err) waitServerUp(t, 7777) @@ -42,7 +42,7 @@ func TestHttpGzipResponse(t *testing.T) { // Create a new mock Http server with custom functions ctrl := gomock.NewController(t) mockHttpServer := mocks_booster_http.NewMockHttpServerApi(ctrl) - httpServer := NewHttpServer("", 7777, mockHttpServer, nil) + httpServer := NewHttpServer("", "0.0.0.0", 7777, mockHttpServer, nil) err := httpServer.Start(context.Background()) require.NoError(t, err) waitServerUp(t, 7777) @@ -109,7 +109,7 @@ func TestHttpInfo(t *testing.T) { // Create a new mock Http server ctrl := gomock.NewController(t) - httpServer := NewHttpServer("", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil) + httpServer := NewHttpServer("", "0.0.0.0", 7777, mocks_booster_http.NewMockHttpServerApi(ctrl), nil) err := httpServer.Start(context.Background()) require.NoError(t, err) waitServerUp(t, 7777) diff --git a/cmd/booster-http/mocks/mock_booster_http.go b/cmd/booster-http/mocks/mock_booster_http.go index 1af9d3451..4f1eca760 100644 --- a/cmd/booster-http/mocks/mock_booster_http.go +++ b/cmd/booster-http/mocks/mock_booster_http.go @@ -8,8 +8,8 @@ import ( context "context" reflect "reflect" - mount "github.com/filecoin-project/dagstore/mount" piecestore "github.com/filecoin-project/boost-gfm/piecestore" + mount "github.com/filecoin-project/dagstore/mount" abi "github.com/filecoin-project/go-state-types/abi" gomock "github.com/golang/mock/gomock" cid "github.com/ipfs/go-cid" diff --git a/cmd/booster-http/run.go b/cmd/booster-http/run.go index e9832aaf0..021037cb8 100644 --- a/cmd/booster-http/run.go +++ b/cmd/booster-http/run.go @@ -42,6 +42,12 @@ var runCmd = &cli.Command{ Usage: "the base path at which to run the web server", Value: "", }, + &cli.StringFlag{ + Name: "address", + Aliases: []string{"addr"}, + Usage: "the listen address for the web server", + Value: "0.0.0.0", + }, &cli.UintFlag{ Name: "port", Usage: "the port the web server listens on", @@ -201,14 +207,15 @@ var runCmd = &cli.Command{ sapi := serverApi{ctx: ctx, bapi: bapi, sa: sa} server := NewHttpServer( cctx.String("base-path"), + cctx.String("address"), cctx.Int("port"), sapi, opts, ) // Start the server - log.Infof("Starting booster-http node on port %d with base path '%s'", - cctx.Int("port"), cctx.String("base-path")) + log.Infof("Starting booster-http node on listen address %s and port %d with base path '%s'", + cctx.String("address"), cctx.Int("port"), cctx.String("base-path")) err = server.Start(ctx) if err != nil { return fmt.Errorf("starting http server: %w", err) diff --git a/cmd/booster-http/server.go b/cmd/booster-http/server.go index 8cf321d44..a4d40f355 100644 --- a/cmd/booster-http/server.go +++ b/cmd/booster-http/server.go @@ -42,11 +42,12 @@ type apiVersion struct { } type HttpServer struct { - path string - port int - api HttpServerApi - opts HttpServerOptions - idxPage string + path string + listenAddr string + port int + api HttpServerApi + opts HttpServerOptions + idxPage string ctx context.Context cancel context.CancelFunc @@ -65,11 +66,11 @@ type HttpServerOptions struct { SupportedResponseFormats []string } -func NewHttpServer(path string, port int, api HttpServerApi, opts *HttpServerOptions) *HttpServer { +func NewHttpServer(path string, listenAddr string, port int, api HttpServerApi, opts *HttpServerOptions) *HttpServer { if opts == nil { opts = &HttpServerOptions{ServePieces: true} } - return &HttpServer{path: path, port: port, api: api, opts: *opts, idxPage: parseTemplate(*opts)} + return &HttpServer{path: path, listenAddr: listenAddr, port: port, api: api, opts: *opts, idxPage: parseTemplate(*opts)} } func (s *HttpServer) pieceBasePath() string { @@ -102,7 +103,7 @@ func (s *HttpServer) Start(ctx context.Context) error { handler.HandleFunc("/info", s.handleInfo) handler.Handle("/metrics", metrics.Exporter("booster_http")) // metrics s.server = &http.Server{ - Addr: fmt.Sprintf(":%d", s.port), + Addr: fmt.Sprintf("%s:%d", s.listenAddr, s.port), Handler: handler, // This context will be the parent of the context associated with all // incoming requests From 83a15c85c7f69270c670a6c8611cd5f962f6cbff Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Fri, 26 May 2023 19:31:28 +0400 Subject: [PATCH 08/15] fix nil ptr (#1470) --- gql/resolver_mpool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gql/resolver_mpool.go b/gql/resolver_mpool.go index f60a4b9db..9860ffed0 100644 --- a/gql/resolver_mpool.go +++ b/gql/resolver_mpool.go @@ -68,7 +68,7 @@ func (r *resolver) Mpool(ctx context.Context, args struct{ Local bool }) ([]*msg method := m.Message.Method.String() toact, err := r.fullNode.StateGetActor(ctx, m.Message.To, types.EmptyTSK) if err == nil { - method = consensus.NewActorRegistry().Methods[toact.Code][m.Message.Method].Params.Name() + method = consensus.NewActorRegistry().Methods[toact.Code][m.Message.Method].Name } var params string From d4acbed4f21a02be044a4b22713582e548e0b693 Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Mon, 29 May 2023 13:44:43 +0400 Subject: [PATCH 09/15] fix: incorrect check when import offline deal data using proposal CID (#1473) * fix incorrect early check * update error msg --- cmd/boostd/import_data.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/boostd/import_data.go b/cmd/boostd/import_data.go index 92aed947e..bed8bcb33 100644 --- a/cmd/boostd/import_data.go +++ b/cmd/boostd/import_data.go @@ -67,9 +67,6 @@ var importDataCmd = &cli.Command{ // If the user has supplied a signed proposal cid deleteAfterImport := cctx.Bool("delete-after-import") if proposalCid != nil { - if deleteAfterImport { - return fmt.Errorf("legacy deal data cannot be automatically deleted after import (only new deals)") - } // Look up the deal in the boost database deal, err := napi.BoostDealBySignedProposalCid(cctx.Context, *proposalCid) @@ -80,6 +77,10 @@ var importDataCmd = &cli.Command{ return err } + if deleteAfterImport { + return fmt.Errorf("cannot find boost deal with proposal cid %s and legacy deal data cannot be automatically deleted after import (only new deals)", proposalCid) + } + // The deal is not in the boost database, try the legacy // markets datastore (v1.1.0 deal) err := napi.MarketImportDealData(cctx.Context, *proposalCid, filePath) From 592a7016bb5a1d6a86f554a03c26ba5d6f2f75fe Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Tue, 30 May 2023 00:33:07 -0700 Subject: [PATCH 10/15] fix(server): properly cancel graphsync requests (#1475) --- retrievalmarket/server/channelstate.go | 2 ++ retrievalmarket/server/gsunpaidretrieval.go | 11 +++++++++++ retrievalmarket/server/gsunpaidretrieval_test.go | 4 ++++ 3 files changed, 17 insertions(+) diff --git a/retrievalmarket/server/channelstate.go b/retrievalmarket/server/channelstate.go index 2a7dbb4a7..1f1b48ed8 100644 --- a/retrievalmarket/server/channelstate.go +++ b/retrievalmarket/server/channelstate.go @@ -4,6 +4,7 @@ import ( "bytes" "github.com/filecoin-project/boost-gfm/retrievalmarket" + graphsync "github.com/filecoin-project/boost-graphsync" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" @@ -22,6 +23,7 @@ type retrievalState struct { retType RetrievalType cs *channelState mkts *retrievalmarket.ProviderDealState + gsReq graphsync.RequestID } func (r retrievalState) ChannelState() channelState { return *r.cs } diff --git a/retrievalmarket/server/gsunpaidretrieval.go b/retrievalmarket/server/gsunpaidretrieval.go index 76d591e8f..2f97a6289 100644 --- a/retrievalmarket/server/gsunpaidretrieval.go +++ b/retrievalmarket/server/gsunpaidretrieval.go @@ -181,8 +181,18 @@ func (g *GraphsyncUnpaidRetrieval) CancelTransfer(ctx context.Context, id datatr rcpt := state.cs.recipient tID := state.cs.transferID + gsRequestID := state.gsReq g.activeRetrievalsLk.Unlock() + // tell GraphSync to cancel the request + if (gsRequestID != graphsync.RequestID{}) { + err := g.Cancel(ctx, gsRequestID) + if err != nil { + log.Info("unable to force close graphsync request %s: %s", tID, err) + } + } + + // send a message on data transfer err := g.dtnet.SendMessage(ctx, rcpt, message.CancelResponse(tID)) g.failTransfer(state, errors.New("transfer cancelled by provider")) @@ -326,6 +336,7 @@ func (g *GraphsyncUnpaidRetrieval) handleRetrievalDeal(peerID peer.ID, msg datat retType: retType, cs: cs, mkts: mktsState, + gsReq: request.ID(), } // Record the data transfer ID so that we can intercept future diff --git a/retrievalmarket/server/gsunpaidretrieval_test.go b/retrievalmarket/server/gsunpaidretrieval_test.go index 8b2395472..758bff251 100644 --- a/retrievalmarket/server/gsunpaidretrieval_test.go +++ b/retrievalmarket/server/gsunpaidretrieval_test.go @@ -319,6 +319,10 @@ func runRequestTest(t *testing.T, tc testCase) { } else { require.NoError(t, err) } + + // final verification -- the server has no active graphsync requests + stats := gsupr.Stats() + require.Equal(t, stats.IncomingRequests.Active, uint64(0)) } func createRetrievalProvider(ctx context.Context, t *testing.T, testData *tut.Libp2pTestData, pieceStore *tut.TestPieceStore, sectorAccessor *testnodes.TestSectorAccessor, dagstoreWrapper *tut.MockDagStoreWrapper, gs graphsync.GraphExchange, paymentAddress address.Address) retrievalmarket.RetrievalProvider { From 9e0581019a7dbd9982bc7106b6b6bff61c9dd164 Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Tue, 30 May 2023 13:39:49 +0400 Subject: [PATCH 11/15] set UI default listen address to localhost (#1476) --- node/config/def.go | 2 +- node/config/doc_gen.go | 2 +- node/config/types.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/node/config/def.go b/node/config/def.go index 911177af0..a81ce1b98 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -65,7 +65,7 @@ func DefaultBoost() *Boost { }, Graphql: GraphqlConfig{ - ListenAddress: "0.0.0.0", + ListenAddress: "127.0.0.1", Port: 8080, }, diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 314596306..33ef71442 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -427,7 +427,7 @@ for any other deal.`, Name: "ListenAddress", Type: "string", - Comment: `The ip address the GraphQL server will bind to. Default: 0.0.0.0`, + Comment: `The ip address the GraphQL server will bind to. Default: 127.0.0.1`, }, { Name: "Port", diff --git a/node/config/types.go b/node/config/types.go index d8c1c5b89..df9e31d41 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -73,7 +73,7 @@ type WalletsConfig struct { } type GraphqlConfig struct { - // The ip address the GraphQL server will bind to. Default: 0.0.0.0 + // The ip address the GraphQL server will bind to. Default: 127.0.0.1 ListenAddress string // The port that the graphql server listens on Port uint64 From f31a21870c723a52986428dc8e2b24f91798281d Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Tue, 30 May 2023 14:43:41 +0400 Subject: [PATCH 12/15] feat: display msg params in the mpool UI (#1471) * show msg params * fix: mpool nil pointer * fix width --------- Co-authored-by: Dirk McCormick --- gql/resolver_mpool.go | 114 +++++++----------------------------------- react/src/Mpool.css | 15 ++++++ react/src/Mpool.js | 4 +- 3 files changed, 37 insertions(+), 96 deletions(-) diff --git a/gql/resolver_mpool.go b/gql/resolver_mpool.go index 9860ffed0..7e2db36f8 100644 --- a/gql/resolver_mpool.go +++ b/gql/resolver_mpool.go @@ -4,17 +4,16 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" + "reflect" "github.com/filecoin-project/lotus/chain/consensus" + cbg "github.com/whyrusleeping/cbor-gen" gqltypes "github.com/filecoin-project/boost/gql/types" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - stbig "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin/multisig" ) type msg struct { @@ -65,22 +64,24 @@ func (r *resolver) Mpool(ctx context.Context, args struct{ Local bool }) ([]*msg } } - method := m.Message.Method.String() + var params string + methodName := m.Message.Method.String() toact, err := r.fullNode.StateGetActor(ctx, m.Message.To, types.EmptyTSK) if err == nil { - method = consensus.NewActorRegistry().Methods[toact.Code][m.Message.Method].Name - } - - var params string - paramsMsg, err := messageFromBytes(m.Message.Params) - if err != nil { - params = err.Error() - } else { - paramsBytes, err := json.MarshalIndent(paramsMsg, "", " ") - if err != nil { - params = err.Error() - } else { - params = string(paramsBytes) + method, ok := consensus.NewActorRegistry().Methods[toact.Code][m.Message.Method] + if ok { + methodName = method.Name + + params = string(m.Message.Params) + p, ok := reflect.New(method.Params.Elem()).Interface().(cbg.CBORUnmarshaler) + if ok { + if err := p.UnmarshalCBOR(bytes.NewReader(m.Message.Params)); err == nil { + b, err := json.MarshalIndent(p, "", " ") + if err == nil { + params = string(b) + } + } + } } } @@ -92,7 +93,7 @@ func (r *resolver) Mpool(ctx context.Context, args struct{ Local bool }) ([]*msg GasFeeCap: gqltypes.BigInt{Int: m.Message.GasFeeCap}, GasLimit: gqltypes.Uint64(uint64(m.Message.GasLimit)), GasPremium: gqltypes.BigInt{Int: m.Message.GasPremium}, - Method: method, + Method: methodName, Params: params, BaseFee: gqltypes.BigInt{Int: baseFee}, }) @@ -101,83 +102,6 @@ func (r *resolver) Mpool(ctx context.Context, args struct{ Local bool }) ([]*msg return gqlmsgs, nil } -func messageFromBytes(msgb []byte) (types.ChainMsg, error) { - // Signed - { - var msg types.SignedMessage - if err := msg.UnmarshalCBOR(bytes.NewReader(msgb)); err == nil { - return &msg, nil - } - } - - // Unsigned - { - var msg types.Message - if err := msg.UnmarshalCBOR(bytes.NewReader(msgb)); err == nil { - return &msg, nil - } - } - - // Multisig propose? - { - var pp multisig.ProposeParams - if err := pp.UnmarshalCBOR(bytes.NewReader(msgb)); err == nil { - i, err := address.NewIDAddress(0) - if err != nil { - return nil, err - } - - return &types.Message{ - // Hack(-ish) - Version: 0x6d736967, - From: i, - - To: pp.To, - Value: pp.Value, - - Method: pp.Method, - Params: pp.Params, - - GasFeeCap: stbig.Zero(), - GasPremium: stbig.Zero(), - }, nil - } - } - - // Encoded json??? - { - if msg, err := messageFromJson(msgb); err == nil { - return msg, nil - } - } - - return nil, errors.New("probably not a cbor-serialized message") -} - -func messageFromJson(msgb []byte) (types.ChainMsg, error) { - // Unsigned - { - var msg types.Message - if err := json.Unmarshal(msgb, &msg); err == nil { - if msg.To != address.Undef { - return &msg, nil - } - } - } - - // Signed - { - var msg types.SignedMessage - if err := json.Unmarshal(msgb, &msg); err == nil { - if msg.Message.To != address.Undef { - return &msg, nil - } - } - } - - return nil, errors.New("probably not a json-serialized message") -} - func mockMessages() []*types.SignedMessage { to0, _ := address.NewFromString("f01469945") from0, _ := address.NewFromString("f3uakndzne4lorwykinlitx2d2puuhgburvxw4dpkfskeofmzg33pm7okyzikqe2gzvaqj2k3hpunwayij6haa") diff --git a/react/src/Mpool.css b/react/src/Mpool.css index 588ec5427..2a30e787e 100644 --- a/react/src/Mpool.css +++ b/react/src/Mpool.css @@ -117,4 +117,19 @@ left: 1.5em; border-left: 1px solid #000; height: 0.5em; +} + +.mpool .params{ + width: 1080px; + text-overflow: ellipsis; + cursor: pointer; + word-break: break-all; + overflow: hidden; + white-space: nowrap; +} + +.mpool .params:hover{ + overflow: visible; + white-space: normal; + width: auto; } \ No newline at end of file diff --git a/react/src/Mpool.js b/react/src/Mpool.js index 89950b346..c7b609afe 100644 --- a/react/src/Mpool.js +++ b/react/src/Mpool.js @@ -67,7 +67,9 @@ function MpoolMessage(props) { Params - {msg.Params} + +
{msg.Params}
+ Gas Fee Cap From cd95724dae0d9a5b647ae5ce902250aae06f14bc Mon Sep 17 00:00:00 2001 From: dirkmc Date: Wed, 31 May 2023 10:27:38 +0200 Subject: [PATCH 13/15] Reset read deadline after reading deal proposal message (#1479) * fix: reset read deadline after reading deal proposal message * fix: increase client request deadline --- go.mod | 2 +- storagemarket/lp2pimpl/net.go | 91 ++++++++++++++++++++++++----------- 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index c9c1b7166..e43191fee 100644 --- a/go.mod +++ b/go.mod @@ -320,7 +320,7 @@ require ( github.com/zondax/hid v0.9.1 // indirect github.com/zondax/ledger-go v0.12.1 // indirect go.uber.org/dig v1.15.0 // indirect - go.uber.org/zap v1.24.0 // indirect + go.uber.org/zap v1.24.0 go4.org v0.0.0-20200411211856-f5505b9728dd // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.7.0 // indirect diff --git a/storagemarket/lp2pimpl/net.go b/storagemarket/lp2pimpl/net.go index aae92782d..e281921b2 100644 --- a/storagemarket/lp2pimpl/net.go +++ b/storagemarket/lp2pimpl/net.go @@ -23,6 +23,7 @@ import ( "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/protocol" + "go.uber.org/zap" ) var log = logging.Logger("boost-net") @@ -31,9 +32,19 @@ var propLog = logging.Logger("boost-prop") const DealProtocolv120ID = "/fil/storage/mk/1.2.0" const DealProtocolv121ID = "/fil/storage/mk/1.2.1" const DealStatusV12ProtocolID = "/fil/storage/status/1.2.0" + +// The time limit to read a message from the client when the client opens a stream const providerReadDeadline = 10 * time.Second + +// The time limit to write a response to the client const providerWriteDeadline = 10 * time.Second -const clientReadDeadline = 10 * time.Second + +// The time limit to wait for the provider to send a response to a client's request. +// This includes the time it takes for the provider to process the request and +// send a response. +const clientReadDeadline = 60 * time.Second + +// The time limit to write a message to the provider const clientWriteDeadline = 10 * time.Second // DealClientOption is an option for configuring the libp2p storage deal client @@ -195,34 +206,44 @@ func (p *DealProvider) Stop() { // Called when the client opens a libp2p stream with a new deal proposal func (p *DealProvider) handleNewDealStream(s network.Stream) { - defer s.Close() + start := time.Now() + reqLogUuid := uuid.New() + reqLog := log.With("reqlog-uuid", reqLogUuid.String(), "client-peer", s.Conn().RemotePeer()) + reqLog.Debugw("new deal proposal request") + + defer func() { + err := s.Close() + if err != nil { + reqLog.Infow("closing stream", "err", err) + } + reqLog.Debugw("handled deal proposal request", "duration", time.Since(start).String()) + }() // Set a deadline on reading from the stream so it doesn't hang _ = s.SetReadDeadline(time.Now().Add(providerReadDeadline)) - defer s.SetReadDeadline(time.Time{}) // nolint // Read the deal proposal from the stream var proposal types.DealParams err := proposal.UnmarshalCBOR(s) + _ = s.SetReadDeadline(time.Time{}) // Clear read deadline so conn doesn't get closed if err != nil { - log.Warnw("reading storage deal proposal from stream", "err", err) + reqLog.Warnw("reading storage deal proposal from stream", "err", err) return } - log.Infow("received deal proposal", "id", proposal.DealUUID, "client-peer", s.Conn().RemotePeer()) + reqLog = reqLog.With("id", proposal.DealUUID) + reqLog.Infow("received deal proposal") // Start executing the deal. // Note: This method just waits for the deal to be accepted, it doesn't // wait for deal execution to complete. + startExec := time.Now() res, err := p.prov.ExecuteDeal(context.Background(), &proposal, s.Conn().RemotePeer()) + reqLog.Debugw("processed deal proposal accept") if err != nil { - log.Warnw("deal proposal failed", "id", proposal.DealUUID, "err", err, "reason", res.Reason) + reqLog.Warnw("deal proposal failed", "err", err, "reason", res.Reason) } - // Set a deadline on writing to the stream so it doesn't hang - _ = s.SetWriteDeadline(time.Now().Add(providerWriteDeadline)) - defer s.SetWriteDeadline(time.Time{}) // nolint - // Log the response propLog.Infow("send deal proposal response", "id", proposal.DealUUID, @@ -238,44 +259,60 @@ func (p *DealProvider) handleNewDealStream(s network.Stream) { "start epoch", proposal.ClientDealProposal.Proposal.StartEpoch, "end epoch", proposal.ClientDealProposal.Proposal.EndEpoch, "price per epoch", proposal.ClientDealProposal.Proposal.StoragePricePerEpoch, + "duration", time.Since(startExec).String(), ) _ = p.plDB.InsertLog(p.ctx, proposal, res.Accepted, res.Reason) //nolint:errcheck + // Set a deadline on writing to the stream so it doesn't hang + _ = s.SetWriteDeadline(time.Now().Add(providerWriteDeadline)) + defer s.SetWriteDeadline(time.Time{}) // nolint + // Write the response to the client err = cborutil.WriteCborRPC(s, &types.DealResponse{Accepted: res.Accepted, Message: res.Reason}) if err != nil { - log.Warnw("writing deal response", "id", proposal.DealUUID, "err", err) - return + reqLog.Warnw("writing deal response", "err", err) } } func (p *DealProvider) handleNewDealStatusStream(s network.Stream) { - defer s.Close() + start := time.Now() + reqLogUuid := uuid.New() + reqLog := log.With("reqlog-uuid", reqLogUuid.String(), "client-peer", s.Conn().RemotePeer()) + reqLog.Debugw("new deal status request") + + defer func() { + err := s.Close() + if err != nil { + reqLog.Infow("closing stream", "err", err) + } + reqLog.Debugw("handled deal status request", "duration", time.Since(start).String()) + }() + // Read the deal status request from the stream _ = s.SetReadDeadline(time.Now().Add(providerReadDeadline)) - defer s.SetReadDeadline(time.Time{}) // nolint - var req types.DealStatusRequest err := req.UnmarshalCBOR(s) + _ = s.SetReadDeadline(time.Time{}) // Clear read deadline so conn doesn't get closed if err != nil { - log.Warnw("reading deal status request from stream", "err", err) + reqLog.Warnw("reading deal status request from stream", "err", err) return } - log.Debugw("received deal status request", "id", req.DealUUID, "client-peer", s.Conn().RemotePeer()) + reqLog = reqLog.With("id", req.DealUUID) + reqLog.Debugw("received deal status request") - resp := p.getDealStatus(req) + resp := p.getDealStatus(req, reqLog) + reqLog.Debugw("processed deal status request") // Set a deadline on writing to the stream so it doesn't hang _ = s.SetWriteDeadline(time.Now().Add(providerWriteDeadline)) defer s.SetWriteDeadline(time.Time{}) // nolint if err := cborutil.WriteCborRPC(s, &resp); err != nil { - log.Errorw("failed to write deal status response", "err", err) - return + reqLog.Errorw("failed to write deal status response", "err", err) } } -func (p *DealProvider) getDealStatus(req types.DealStatusRequest) types.DealStatusResponse { +func (p *DealProvider) getDealStatus(req types.DealStatusRequest, reqLog *zap.SugaredLogger) types.DealStatusResponse { errResp := func(err string) types.DealStatusResponse { return types.DealStatusResponse{DealUUID: req.DealUUID, Error: err} } @@ -286,34 +323,34 @@ func (p *DealProvider) getDealStatus(req types.DealStatusRequest) types.DealStat } if err != nil { - log.Errorw("failed to fetch deal status", "err", err) + reqLog.Errorw("failed to fetch deal status", "err", err) return errResp("failed to fetch deal status") } // verify request signature uuidBytes, err := req.DealUUID.MarshalBinary() if err != nil { - log.Errorw("failed to serialize request deal UUID", "err", err) + reqLog.Errorw("failed to serialize request deal UUID", "err", err) return errResp("failed to serialize request deal UUID") } clientAddr := pds.ClientDealProposal.Proposal.Client addr, err := p.fullNode.StateAccountKey(p.ctx, clientAddr, chaintypes.EmptyTSK) if err != nil { - log.Errorw("failed to get account key for client addr", "client", clientAddr.String(), "err", err) + reqLog.Errorw("failed to get account key for client addr", "client", clientAddr.String(), "err", err) msg := fmt.Sprintf("failed to get account key for client addr %s", clientAddr.String()) return errResp(msg) } err = sigs.Verify(&req.Signature, addr, uuidBytes) if err != nil { - log.Warnw("signature verification failed", "err", err) + reqLog.Warnw("signature verification failed", "err", err) return errResp("signature verification failed") } signedPropCid, err := pds.SignedProposalCid() if err != nil { - log.Errorw("getting signed proposal cid", "err", err) + reqLog.Errorw("getting signed proposal cid", "err", err) return errResp("getting signed proposal cid") } @@ -321,7 +358,7 @@ func (p *DealProvider) getDealStatus(req types.DealStatusRequest) types.DealStat si, err := p.spApi.SectorsStatus(p.ctx, pds.SectorID, false) if err != nil { - log.Errorw("getting sector status from sealer", "err", err) + reqLog.Errorw("getting sector status from sealer", "err", err) return errResp("getting sector status from sealer") } From 49df3138a8525d4bfe73adf2eac2fd2458528718 Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Wed, 31 May 2023 15:57:57 +0400 Subject: [PATCH 14/15] feat: Show elapsed epoch and PSD wait epochs in UI (#1480) * show epochs * fix devnet UI, use BlockdDelaySecs * fix lint err * Update gql/resolver.go Co-authored-by: dirkmc --------- Co-authored-by: dirkmc --- docker/devnet/boost/entrypoint.sh | 1 + docker/devnet/docker-compose.yaml | 2 +- gql/resolver.go | 10 +++++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docker/devnet/boost/entrypoint.sh b/docker/devnet/boost/entrypoint.sh index f5bf2454e..82a77e7a6 100755 --- a/docker/devnet/boost/entrypoint.sh +++ b/docker/devnet/boost/entrypoint.sh @@ -40,6 +40,7 @@ if [ ! -f $BOOST_PATH/.init.boost ]; then echo Setting port in boost config... sed -i 's|ip4/0.0.0.0/tcp/0|ip4/0.0.0.0/tcp/50000|g' $BOOST_PATH/config.toml + sed -i 's|127.0.0.1|0.0.0.0|g' $BOOST_PATH/config.toml echo Done touch $BOOST_PATH/.init.boost diff --git a/docker/devnet/docker-compose.yaml b/docker/devnet/docker-compose.yaml index 2e890ff45..ccbb8a823 100644 --- a/docker/devnet/docker-compose.yaml +++ b/docker/devnet/docker-compose.yaml @@ -138,4 +138,4 @@ services: environment: HTTP_BIND: "${HTTP_BIND:-127.0.0.1}" HTTP_PORT: "${HTTP_PORT:-4080}" - network_mode: host \ No newline at end of file + network_mode: host diff --git a/gql/resolver.go b/gql/resolver.go index 7195d42e3..26a9f8d7f 100644 --- a/gql/resolver.go +++ b/gql/resolver.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "math" + "time" "github.com/dustin/go-humanize" "github.com/filecoin-project/boost-gfm/piecestore" @@ -26,6 +27,7 @@ import ( "github.com/filecoin-project/boost/transport" "github.com/filecoin-project/dagstore" "github.com/filecoin-project/lotus/api/v1api" + "github.com/filecoin-project/lotus/build" lotus_repo "github.com/filecoin-project/lotus/node/repo" "github.com/google/uuid" "github.com/graph-gophers/graphql-go" @@ -527,14 +529,14 @@ func (dr *dealResolver) Retry() string { } func (dr *dealResolver) Message(ctx context.Context) string { - msg := dr.message(ctx, dr.ProviderDealState.Checkpoint) + msg := dr.message(ctx, dr.ProviderDealState.Checkpoint, dr.ProviderDealState.CheckpointAt) if dr.ProviderDealState.Retry != types.DealRetryFatal && dr.ProviderDealState.Err != "" { msg = "Paused at '" + msg + "': " + dr.ProviderDealState.Err } return msg } -func (dr *dealResolver) message(ctx context.Context, checkpoint dealcheckpoints.Checkpoint) string { +func (dr *dealResolver) message(ctx context.Context, checkpoint dealcheckpoints.Checkpoint, checkpointAt time.Time) string { switch checkpoint { case dealcheckpoints.Accepted: if dr.IsOffline { @@ -570,7 +572,9 @@ func (dr *dealResolver) message(ctx context.Context, checkpoint dealcheckpoints. case dealcheckpoints.Transferred: return "Ready to Publish" case dealcheckpoints.Published: - return "Awaiting Publish Confirmation" + elapsedEpochs := uint64(time.Since(checkpointAt).Seconds()) / build.BlockDelaySecs + confidenceEpochs := build.MessageConfidence * 2 + return fmt.Sprintf("Awaiting Publish Confirmation (%d/%d epochs)", elapsedEpochs, confidenceEpochs) case dealcheckpoints.PublishConfirmed: return "Adding to Sector" case dealcheckpoints.AddedPiece: From 43ee60325ea1e5cd1f4c4ca2362490e87bba19bf Mon Sep 17 00:00:00 2001 From: LexLuthr <88259624+LexLuthr@users.noreply.github.com> Date: Wed, 31 May 2023 20:35:59 +0400 Subject: [PATCH 15/15] release v1.7.3-rc3 (#1481) --- build/openrpc/boost.json.gz | Bin 5619 -> 5619 bytes build/version.go | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index ccc996f2b2f3f9e91517bf0d09cf9a65e15337ea..056cd03acad5815baf88700533f64592d51a3223 100644 GIT binary patch delta 21 dcmeyY{aJfL3*)?vt!bhhQ?gBFI0-T`003yA2gm>b delta 21 ccmeyY{aJfL3*)+tt!bhh2@><;oCFye0A~CLf&c&j diff --git a/build/version.go b/build/version.go index 112065b5e..dbc1ffc08 100644 --- a/build/version.go +++ b/build/version.go @@ -2,7 +2,7 @@ package build var CurrentCommit string -const BuildVersion = "1.7.3-rc2" +const BuildVersion = "1.7.3-rc3" func UserVersion() string { return BuildVersion + CurrentCommit