diff --git a/CHANGELOG.md b/CHANGELOG.md index f2e34c336b7..d3b9663c131 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Kubo Changelogs +- [v0.16](docs/changelogs/v0.16.md) - [v0.15](docs/changelogs/v0.15.md) - [v0.14](docs/changelogs/v0.14.md) - [v0.13](docs/changelogs/v0.13.md) diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 1b6a8fa20aa..34762e74798 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -59,6 +59,7 @@ const ( routingOptionDHTKwd = "dht" routingOptionDHTServerKwd = "dhtserver" routingOptionNoneKwd = "none" + routingOptionCustomKwd = "custom" routingOptionDefaultKwd = "default" unencryptTransportKwd = "disable-transport-encryption" unrestrictedApiAccessKwd = "unrestricted-api" @@ -401,7 +402,10 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment routingOption, _ := req.Options[routingOptionKwd].(string) if routingOption == routingOptionDefaultKwd { - routingOption = cfg.Routing.Type.WithDefault(routingOptionDHTKwd) + routingOption = cfg.Routing.Type + if routingOption == "" { + routingOption = routingOptionDHTKwd + } } switch routingOption { case routingOptionSupernodeKwd: @@ -414,6 +418,14 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment ncfg.Routing = libp2p.DHTServerOption case routingOptionNoneKwd: ncfg.Routing = libp2p.NilRouterOption + case routingOptionCustomKwd: + ncfg.Routing = libp2p.ConstructDelegatedRouting( + cfg.Routing.Routers, + cfg.Routing.Methods, + cfg.Identity.PeerID, + cfg.Addresses.Swarm, + cfg.Identity.PrivKey, + ) default: return fmt.Errorf("unrecognized routing option: %s", routingOption) } diff --git a/config/init.go b/config/init.go index 6bc7a1785c5..1eca51d31d2 100644 --- a/config/init.go +++ b/config/init.go @@ -48,7 +48,15 @@ func InitWithIdentity(identity Identity) (*Config, error) { }, Routing: Routing{ - Type: NewOptionalString("dht"), + Type: "dht", + Methods: Methods{ + MethodNameFindPeers: Method{}, + MethodNameFindProviders: Method{}, + MethodNameGetIPNS: Method{}, + MethodNameProvide: Method{}, + MethodNamePutIPNS: Method{}, + }, + Routers: nil, }, // setup the node mount points. diff --git a/config/profile.go b/config/profile.go index 8252d1ab91e..cbc7c976453 100644 --- a/config/profile.go +++ b/config/profile.go @@ -174,7 +174,7 @@ functionality - performance of content discovery and data fetching may be degraded. `, Transform: func(c *Config) error { - c.Routing.Type = NewOptionalString("dhtclient") + c.Routing.Type = "dhtclient" c.AutoNAT.ServiceMode = AutoNATServiceDisabled c.Reprovider.Interval = "0" diff --git a/config/routing.go b/config/routing.go index 4a96589daad..cecba7a5edd 100644 --- a/config/routing.go +++ b/config/routing.go @@ -1,28 +1,101 @@ package config +import ( + "encoding/json" + "fmt" +) + // Routing defines configuration options for libp2p routing type Routing struct { // Type sets default daemon routing mode. // - // Can be one of "dht", "dhtclient", "dhtserver", "none", or unset. - Type *OptionalString `json:",omitempty"` + // Can be one of "dht", "dhtclient", "dhtserver", "none", or "custom". + // When "custom" is set, you can specify a list of Routers. + Type string - Routers map[string]Router + Routers Routers + + Methods Methods } type Router struct { - // Currenly only supported Type is "reframe". + // Currenly supported Types are "reframe", "dht", "parallel", "sequential". // Reframe type allows to add other resolvers using the Reframe spec: // https://github.com/ipfs/specs/tree/main/reframe // In the future we will support "dht" and other Types here. - Type string - - Enabled Flag `json:",omitempty"` + Type RouterType // Parameters are extra configuration that this router might need. // A common one for reframe router is "Endpoint". - Parameters map[string]string + Parameters interface{} +} + +type Routers map[string]RouterParser +type Methods map[MethodName]Method + +func (m Methods) Check() error { + + // Check supported methods + for _, mn := range MethodNameList { + _, ok := m[mn] + if !ok { + return fmt.Errorf("method name %q is missing from Routing.Methods config param", mn) + } + } + + // Check unsupported methods + for k := range m { + seen := false + for _, mn := range MethodNameList { + if mn == k { + seen = true + break + } + } + + if seen { + continue + } + + return fmt.Errorf("method name %q is not a supported method on Routing.Methods config param", k) + } + + return nil +} + +type RouterParser struct { + Router +} + +func (r *RouterParser) UnmarshalJSON(b []byte) error { + out := Router{} + out.Parameters = &json.RawMessage{} + if err := json.Unmarshal(b, &out); err != nil { + return err + } + raw := out.Parameters.(*json.RawMessage) + + var p interface{} + switch out.Type { + case RouterTypeReframe: + p = &ReframeRouterParams{} + case RouterTypeDHT: + p = &DHTRouterParams{} + case RouterTypeSequential: + p = &ComposableRouterParams{} + case RouterTypeParallel: + p = &ComposableRouterParams{} + } + + if err := json.Unmarshal(*raw, &p); err != nil { + return err + } + + r.Router.Type = out.Type + r.Router.Parameters = p + + return nil } // Type is the routing type. @@ -30,15 +103,56 @@ type Router struct { type RouterType string const ( - RouterTypeReframe RouterType = "reframe" + RouterTypeReframe RouterType = "reframe" + RouterTypeDHT RouterType = "dht" + RouterTypeSequential RouterType = "sequential" + RouterTypeParallel RouterType = "parallel" ) -type RouterParam string +type DHTMode string const ( - // RouterParamEndpoint is the URL where the routing implementation will point to get the information. - // Usually used for reframe Routers. - RouterParamEndpoint RouterParam = "Endpoint" + DHTModeServer DHTMode = "server" + DHTModeClient DHTMode = "client" + DHTModeAuto DHTMode = "auto" +) - RouterParamPriority RouterParam = "Priority" +type MethodName string + +const ( + MethodNameProvide MethodName = "provide" + MethodNameFindProviders MethodName = "find-providers" + MethodNameFindPeers MethodName = "find-peers" + MethodNameGetIPNS MethodName = "get-ipns" + MethodNamePutIPNS MethodName = "put-ipns" ) + +var MethodNameList = []MethodName{MethodNameProvide, MethodNameFindPeers, MethodNameFindProviders, MethodNameGetIPNS, MethodNamePutIPNS} + +type ReframeRouterParams struct { + // Endpoint is the URL where the routing implementation will point to get the information. + // Usually used for reframe Routers. + Endpoint string +} + +type DHTRouterParams struct { + Mode DHTMode + AcceleratedDHTClient bool `json:",omitempty"` + PublicIPNetwork bool +} + +type ComposableRouterParams struct { + Routers []ConfigRouter + Timeout *OptionalDuration `json:",omitempty"` +} + +type ConfigRouter struct { + RouterName string + Timeout Duration + IgnoreErrors bool + ExecuteAfter *OptionalDuration `json:",omitempty"` +} + +type Method struct { + RouterName string +} diff --git a/config/routing_test.go b/config/routing_test.go new file mode 100644 index 00000000000..013e285a5db --- /dev/null +++ b/config/routing_test.go @@ -0,0 +1,195 @@ +package config + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestRouterParameters(t *testing.T) { + require := require.New(t) + sec := time.Second + min := time.Minute + r := Routing{ + Type: "custom", + Routers: map[string]RouterParser{ + "router-dht": {Router{ + Type: RouterTypeDHT, + Parameters: DHTRouterParams{ + Mode: "auto", + AcceleratedDHTClient: true, + PublicIPNetwork: false, + }, + }}, + "router-reframe": {Router{ + Type: RouterTypeReframe, + Parameters: ReframeRouterParams{ + Endpoint: "reframe-endpoint", + }, + }}, + "router-parallel": {Router{ + Type: RouterTypeParallel, + Parameters: ComposableRouterParams{ + Routers: []ConfigRouter{ + { + RouterName: "router-dht", + Timeout: Duration{10 * time.Second}, + IgnoreErrors: true, + }, + { + RouterName: "router-reframe", + Timeout: Duration{10 * time.Second}, + IgnoreErrors: false, + ExecuteAfter: &OptionalDuration{&sec}, + }, + }, + Timeout: &OptionalDuration{&min}, + }}, + }, + "router-sequential": {Router{ + Type: RouterTypeSequential, + Parameters: ComposableRouterParams{ + Routers: []ConfigRouter{ + { + RouterName: "router-dht", + Timeout: Duration{10 * time.Second}, + IgnoreErrors: true, + }, + { + RouterName: "router-reframe", + Timeout: Duration{10 * time.Second}, + IgnoreErrors: false, + }, + }, + Timeout: &OptionalDuration{&min}, + }}, + }, + }, + Methods: Methods{ + MethodNameFindPeers: { + RouterName: "router-reframe", + }, + MethodNameFindProviders: { + RouterName: "router-dht", + }, + MethodNameGetIPNS: { + RouterName: "router-sequential", + }, + MethodNameProvide: { + RouterName: "router-parallel", + }, + MethodNamePutIPNS: { + RouterName: "router-parallel", + }, + }, + } + + out, err := json.Marshal(r) + require.NoError(err) + + r2 := &Routing{} + + err = json.Unmarshal(out, r2) + require.NoError(err) + + require.Equal(5, len(r2.Methods)) + + dhtp := r2.Routers["router-dht"].Parameters + require.IsType(&DHTRouterParams{}, dhtp) + + rp := r2.Routers["router-reframe"].Parameters + require.IsType(&ReframeRouterParams{}, rp) + + sp := r2.Routers["router-sequential"].Parameters + require.IsType(&ComposableRouterParams{}, sp) + + pp := r2.Routers["router-parallel"].Parameters + require.IsType(&ComposableRouterParams{}, pp) +} + +func TestRouterMissingParameters(t *testing.T) { + require := require.New(t) + + r := Routing{ + Type: "custom", + Routers: map[string]RouterParser{ + "router-wrong-reframe": {Router{ + Type: RouterTypeReframe, + Parameters: DHTRouterParams{ + Mode: "auto", + AcceleratedDHTClient: true, + PublicIPNetwork: false, + }, + }}, + }, + Methods: Methods{ + MethodNameFindPeers: { + RouterName: "router-wrong-reframe", + }, + MethodNameFindProviders: { + RouterName: "router-wrong-reframe", + }, + MethodNameGetIPNS: { + RouterName: "router-wrong-reframe", + }, + MethodNameProvide: { + RouterName: "router-wrong-reframe", + }, + MethodNamePutIPNS: { + RouterName: "router-wrong-reframe", + }, + }, + } + + out, err := json.Marshal(r) + require.NoError(err) + + r2 := &Routing{} + + err = json.Unmarshal(out, r2) + require.NoError(err) + require.Empty(r2.Routers["router-wrong-reframe"].Parameters.(*ReframeRouterParams).Endpoint) +} + +func TestMethods(t *testing.T) { + require := require.New(t) + + methodsOK := Methods{ + MethodNameFindPeers: { + RouterName: "router-wrong-reframe", + }, + MethodNameFindProviders: { + RouterName: "router-wrong-reframe", + }, + MethodNameGetIPNS: { + RouterName: "router-wrong-reframe", + }, + MethodNameProvide: { + RouterName: "router-wrong-reframe", + }, + MethodNamePutIPNS: { + RouterName: "router-wrong-reframe", + }, + } + + require.NoError(methodsOK.Check()) + + methodsMissing := Methods{ + MethodNameFindPeers: { + RouterName: "router-wrong-reframe", + }, + MethodNameGetIPNS: { + RouterName: "router-wrong-reframe", + }, + MethodNameProvide: { + RouterName: "router-wrong-reframe", + }, + MethodNamePutIPNS: { + RouterName: "router-wrong-reframe", + }, + } + + require.Error(methodsMissing.Check()) +} diff --git a/config/types.go b/config/types.go index f27c5fa13e9..e3f61546b51 100644 --- a/config/types.go +++ b/config/types.go @@ -262,6 +262,38 @@ func (d OptionalDuration) String() string { var _ json.Unmarshaler = (*OptionalDuration)(nil) var _ json.Marshaler = (*OptionalDuration)(nil) +type Duration struct { + time.Duration +} + +func (d Duration) MarshalJSON() ([]byte, error) { + return json.Marshal(d.String()) +} + +func (d *Duration) UnmarshalJSON(b []byte) error { + var v interface{} + if err := json.Unmarshal(b, &v); err != nil { + return err + } + switch value := v.(type) { + case float64: + d.Duration = time.Duration(value) + return nil + case string: + var err error + d.Duration, err = time.ParseDuration(value) + if err != nil { + return err + } + return nil + default: + return fmt.Errorf("unable to parse duration, expected a duration string or a float, but got %T", v) + } +} + +var _ json.Unmarshaler = (*Duration)(nil) +var _ json.Marshaler = (*Duration)(nil) + // OptionalInteger represents an integer that has a default value // // When encoded in json, Default is encoded as "null" diff --git a/core/core.go b/core/core.go index 17ccc088e5b..9e515b8162d 100644 --- a/core/core.go +++ b/core/core.go @@ -87,18 +87,18 @@ type IpfsNode struct { RecordValidator record.Validator // Online - PeerHost p2phost.Host `optional:"true"` // the network host (server+client) - Peering *peering.PeeringService `optional:"true"` - Filters *ma.Filters `optional:"true"` - Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper - Routing irouting.TieredRouter `optional:"true"` // the routing system. recommend ipfs-dht - DNSResolver *madns.Resolver // the DNS resolver - Exchange exchange.Interface // the block exchange + strategy (bitswap) - Namesys namesys.NameSystem // the name system, resolves paths to hashes - Provider provider.System // the value provider system - IpnsRepub *ipnsrp.Republisher `optional:"true"` - GraphExchange graphsync.GraphExchange `optional:"true"` - ResourceManager network.ResourceManager `optional:"true"` + PeerHost p2phost.Host `optional:"true"` // the network host (server+client) + Peering *peering.PeeringService `optional:"true"` + Filters *ma.Filters `optional:"true"` + Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper + Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht + DNSResolver *madns.Resolver // the DNS resolver + Exchange exchange.Interface // the block exchange + strategy (bitswap) + Namesys namesys.NameSystem // the name system, resolves paths to hashes + Provider provider.System // the value provider system + IpnsRepub *ipnsrp.Republisher `optional:"true"` + GraphExchange graphsync.GraphExchange `optional:"true"` + ResourceManager network.ResourceManager `optional:"true"` PubSub *pubsub.PubSub `optional:"true"` PSRouter *psrouter.PubsubValueStore `optional:"true"` diff --git a/core/core_test.go b/core/core_test.go index 71de31de613..9a9075ae726 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -117,8 +117,6 @@ func TestDelegatedRoutingSingle(t *testing.T) { err = n.Routing.PutValue(ctx, theID, v) require.NoError(err) - err = n.Routing.PutValue(ctx, theErrorID, v) - require.Error(err) } func TestDelegatedRoutingMulti(t *testing.T) { @@ -164,12 +162,6 @@ func TestDelegatedRoutingMulti(t *testing.T) { require.NoError(err) require.NotNil(v) require.Contains(string(v), "RECORD FROM SERVICE 2") - - err = n.Routing.PutValue(ctx, theID1, v) - require.Error(err) - - err = n.Routing.PutValue(ctx, theID2, v) - require.Error(err) } func StartRoutingServer(t *testing.T, d drs.DelegatedRoutingService) string { @@ -187,16 +179,41 @@ func StartRoutingServer(t *testing.T, d drs.DelegatedRoutingService) string { func GetNode(t *testing.T, reframeURLs ...string) *IpfsNode { t.Helper() - routers := make(map[string]config.Router) + routers := make(config.Routers) + var routerNames []string for i, ru := range reframeURLs { - routers[fmt.Sprintf("reframe-%d", i)] = config.Router{ - Type: string(config.RouterTypeReframe), - Parameters: map[string]string{ - string(config.RouterParamEndpoint): ru, - }, - } + rn := fmt.Sprintf("reframe-%d", i) + routerNames = append(routerNames, rn) + routers[rn] = + config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeReframe, + Parameters: &config.ReframeRouterParams{ + Endpoint: ru, + }, + }, + } + } + + var crs []config.ConfigRouter + for _, rn := range routerNames { + crs = append(crs, config.ConfigRouter{ + RouterName: rn, + IgnoreErrors: true, + Timeout: config.Duration{Duration: time.Minute}, + }) } + const parallelRouterName = "parallel-router" + + routers[parallelRouterName] = config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeParallel, + Parameters: &config.ComposableRouterParams{ + Routers: crs, + }, + }, + } cfg := config.Config{ Identity: testIdentity, Addresses: config.Addresses{ @@ -204,8 +221,25 @@ func GetNode(t *testing.T, reframeURLs ...string) *IpfsNode { API: []string{"/ip4/127.0.0.1/tcp/0"}, }, Routing: config.Routing{ - Type: config.NewOptionalString("none"), + Type: "custom", Routers: routers, + Methods: config.Methods{ + config.MethodNameFindPeers: config.Method{ + RouterName: parallelRouterName, + }, + config.MethodNameFindProviders: config.Method{ + RouterName: parallelRouterName, + }, + config.MethodNameGetIPNS: config.Method{ + RouterName: parallelRouterName, + }, + config.MethodNameProvide: config.Method{ + RouterName: parallelRouterName, + }, + config.MethodNamePutIPNS: config.Method{ + RouterName: parallelRouterName, + }, + }, }, } @@ -214,7 +248,19 @@ func GetNode(t *testing.T, reframeURLs ...string) *IpfsNode { D: syncds.MutexWrap(datastore.NewMapDatastore()), } - n, err := NewNode(context.Background(), &BuildCfg{Repo: r, Online: true, Routing: libp2p.NilRouterOption}) + n, err := NewNode(context.Background(), + &BuildCfg{ + Repo: r, + Online: true, + Routing: libp2p.ConstructDelegatedRouting( + cfg.Routing.Routers, + cfg.Routing.Methods, + cfg.Identity.PeerID, + cfg.Addresses.Swarm, + cfg.Identity.PrivKey, + ), + }, + ) require.NoError(t, err) return n @@ -240,6 +286,10 @@ func (drs *delegatedRoutingService) FindProviders(ctx context.Context, key cid.C return nil, errNotSupported } +func (drs *delegatedRoutingService) Provide(ctx context.Context, req *client.ProvideRequest) (<-chan client.ProvideAsyncResult, error) { + return nil, errNotSupported +} + func (drs *delegatedRoutingService) GetIPNS(ctx context.Context, id []byte) (<-chan client.GetIPNSAsyncResult, error) { ctx, cancel := context.WithCancel(ctx) ch := make(chan client.GetIPNSAsyncResult) diff --git a/core/node/bitswap.go b/core/node/bitswap.go index a0d47cdfc78..2b9cf641a1b 100644 --- a/core/node/bitswap.go +++ b/core/node/bitswap.go @@ -55,7 +55,7 @@ type onlineExchangeIn struct { Mctx helpers.MetricsCtx Host host.Host - Rt irouting.TieredRouter + Rt irouting.ProvideManyRouter Bs blockstore.GCBlockstore BitswapOpts []bitswap.Option `group:"bitswap-options"` } diff --git a/core/node/groups.go b/core/node/groups.go index 0a555892c38..fca984650d9 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -165,7 +165,6 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config) fx.Option { fx.Provide(libp2p.ContentRouting), fx.Provide(libp2p.BaseRouting(cfg.Experimental.AcceleratedDHTClient)), - fx.Provide(libp2p.DelegatedRouting(cfg.Routing.Routers)), maybeProvide(libp2p.PubsubRouter, bcfg.getOpt("ipnsps")), maybeProvide(libp2p.BandwidthCounter, !cfg.Swarm.DisableBandwidthMetrics), diff --git a/core/node/ipns.go b/core/node/ipns.go index 36f330b9b15..0500c7be485 100644 --- a/core/node/ipns.go +++ b/core/node/ipns.go @@ -11,11 +11,10 @@ import ( "github.com/libp2p/go-libp2p/core/peerstore" madns "github.com/multiformats/go-multiaddr-dns" - irouting "github.com/ipfs/kubo/routing" - "github.com/ipfs/go-namesys" "github.com/ipfs/go-namesys/republisher" "github.com/ipfs/kubo/repo" + irouting "github.com/ipfs/kubo/routing" ) const DefaultIpnsCacheSize = 128 @@ -29,8 +28,8 @@ func RecordValidator(ps peerstore.Peerstore) record.Validator { } // Namesys creates new name system -func Namesys(cacheSize int) func(rt irouting.TieredRouter, rslv *madns.Resolver, repo repo.Repo) (namesys.NameSystem, error) { - return func(rt irouting.TieredRouter, rslv *madns.Resolver, repo repo.Repo) (namesys.NameSystem, error) { +func Namesys(cacheSize int) func(rt irouting.ProvideManyRouter, rslv *madns.Resolver, repo repo.Repo) (namesys.NameSystem, error) { + return func(rt irouting.ProvideManyRouter, rslv *madns.Resolver, repo repo.Repo) (namesys.NameSystem, error) { opts := []namesys.Option{ namesys.WithDatastore(repo.Datastore()), namesys.WithDNSResolver(rslv), diff --git a/core/node/libp2p/routing.go b/core/node/libp2p/routing.go index 4825bbc12db..21afe292d24 100644 --- a/core/node/libp2p/routing.go +++ b/core/node/libp2p/routing.go @@ -129,39 +129,6 @@ func BaseRouting(experimentalDHTClient bool) interface{} { } } -type delegatedRouterOut struct { - fx.Out - - Routers []Router `group:"routers,flatten"` - ContentRouter []routing.ContentRouting `group:"content-routers,flatten"` -} - -func DelegatedRouting(routers map[string]config.Router) interface{} { - return func() (delegatedRouterOut, error) { - out := delegatedRouterOut{} - - for _, v := range routers { - if !v.Enabled.WithDefault(true) { - continue - } - - r, err := irouting.RoutingFromConfig(v) - if err != nil { - return out, err - } - - out.Routers = append(out.Routers, Router{ - Routing: r, - Priority: irouting.GetPriority(v.Parameters), - }) - - out.ContentRouter = append(out.ContentRouter, r) - } - - return out, nil - } -} - type p2pOnlineContentRoutingIn struct { fx.In @@ -195,24 +162,23 @@ type p2pOnlineRoutingIn struct { // Routing will get all routers obtained from different methods // (delegated routers, pub-sub, and so on) and add them all together // using a TieredRouter. -func Routing(in p2pOnlineRoutingIn) irouting.TieredRouter { +func Routing(in p2pOnlineRoutingIn) irouting.ProvideManyRouter { routers := in.Routers sort.SliceStable(routers, func(i, j int) bool { return routers[i].Priority < routers[j].Priority }) - irouters := make([]routing.Routing, len(routers)) - for i, v := range routers { - irouters[i] = v.Routing + var cRouters []*routinghelpers.ParallelRouter + for _, v := range routers { + cRouters = append(cRouters, &routinghelpers.ParallelRouter{ + Timeout: 5 * time.Minute, + IgnoreError: true, + Router: v.Routing, + }) } - return irouting.Tiered{ - Tiered: routinghelpers.Tiered{ - Routers: irouters, - Validator: in.Validator, - }, - } + return routinghelpers.NewComposableParallel(cRouters) } // OfflineRouting provides a special Router to the routers list when we are creating a offline node. diff --git a/core/node/libp2p/routingopt.go b/core/node/libp2p/routingopt.go index 2e0ac4ab6a9..262501c69b1 100644 --- a/core/node/libp2p/routingopt.go +++ b/core/node/libp2p/routingopt.go @@ -2,7 +2,10 @@ package libp2p import ( "context" + "github.com/ipfs/go-datastore" + "github.com/ipfs/kubo/config" + irouting "github.com/ipfs/kubo/routing" dht "github.com/libp2p/go-libp2p-kad-dht" dual "github.com/libp2p/go-libp2p-kad-dht/dual" record "github.com/libp2p/go-libp2p-record" @@ -46,6 +49,36 @@ func constructDHTRouting(mode dht.ModeOpt) func( } } +func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs []string, privKey string) func( + ctx context.Context, + host host.Host, + dstore datastore.Batching, + validator record.Validator, + bootstrapPeers ...peer.AddrInfo, +) (routing.Routing, error) { + return func( + ctx context.Context, + host host.Host, + dstore datastore.Batching, + validator record.Validator, + bootstrapPeers ...peer.AddrInfo, + ) (routing.Routing, error) { + return irouting.Parse(routers, methods, + &irouting.ExtraDHTParams{ + BootstrapPeers: bootstrapPeers, + Host: host, + Validator: validator, + Datastore: dstore, + Context: ctx, + }, + &irouting.ExtraReframeParams{ + PeerID: peerID, + Addrs: addrs, + PrivKeyB64: privKey, + }) + } +} + func constructNilRouting( ctx context.Context, host host.Host, diff --git a/core/node/provider.go b/core/node/provider.go index 52cc0b074d7..cf625b8c686 100644 --- a/core/node/provider.go +++ b/core/node/provider.go @@ -28,13 +28,13 @@ func ProviderQueue(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo) (*q } // SimpleProvider creates new record provider -func SimpleProvider(mctx helpers.MetricsCtx, lc fx.Lifecycle, queue *q.Queue, rt irouting.TieredRouter) provider.Provider { +func SimpleProvider(mctx helpers.MetricsCtx, lc fx.Lifecycle, queue *q.Queue, rt irouting.ProvideManyRouter) provider.Provider { return simple.NewProvider(helpers.LifecycleCtx(mctx, lc), queue, rt) } // SimpleReprovider creates new reprovider func SimpleReprovider(reproviderInterval time.Duration) interface{} { - return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, rt irouting.TieredRouter, keyProvider simple.KeyChanFunc) (provider.Reprovider, error) { + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, rt irouting.ProvideManyRouter, keyProvider simple.KeyChanFunc) (provider.Reprovider, error) { return simple.NewReprovider(helpers.LifecycleCtx(mctx, lc), reproviderInterval, rt, keyProvider), nil } } @@ -62,12 +62,7 @@ func SimpleProviderSys(isOnline bool) interface{} { // BatchedProviderSys creates new provider system func BatchedProviderSys(isOnline bool, reprovideInterval string) interface{} { - return func(lc fx.Lifecycle, cr irouting.TieredRouter, q *q.Queue, keyProvider simple.KeyChanFunc, repo repo.Repo) (provider.System, error) { - r := cr.ProvideMany() - if r == nil { - return nil, fmt.Errorf("BatchedProviderSys requires a content router that supports provideMany") - } - + return func(lc fx.Lifecycle, cr irouting.ProvideManyRouter, q *q.Queue, keyProvider simple.KeyChanFunc, repo repo.Repo) (provider.System, error) { reprovideIntervalDuration := kReprovideFrequency if reprovideInterval != "" { dur, err := time.ParseDuration(reprovideInterval) @@ -78,7 +73,7 @@ func BatchedProviderSys(isOnline bool, reprovideInterval string) interface{} { reprovideIntervalDuration = dur } - sys, err := batched.New(r, q, + sys, err := batched.New(cr, q, batched.ReproviderInterval(reprovideIntervalDuration), batched.Datastore(repo.Datastore()), batched.KeyProvider(keyProvider)) diff --git a/docs/changelogs/v0.16.md b/docs/changelogs/v0.16.md new file mode 100644 index 00000000000..558d93d50db --- /dev/null +++ b/docs/changelogs/v0.16.md @@ -0,0 +1,94 @@ +# Kubo changelog v0.16 + +## v0.16.0 + +### Overview + +Below is an outline of all that is in this release, so you get a sense of all that's included. + +- [Kubo changelog v0.16](#kubo-changelog-v016) + - [v0.16.0](#v0160) + - [Overview](#overview) + - [🔦 Highlights](#-highlights) + - [🛣️ More configurable delegated routing system](#️-more-configurable-delegated-routing-system) + - [Changelog](#changelog) + - [Contributors](#contributors) + + +### 🔦 Highlights + + + +#### 🛣️ More configurable delegated routing system + +Since Kubo v0.14.0 [Reframe protocol](https://github.com/ipfs/specs/tree/main/reframe#readme) has been supported as a new routing system. + +Now, we allow to configure several routers working together, so you can have several `reframe` and `dht` routers making queries. You can use the special `parallel` and `sequential` routers to fill your needs. + +Example configuration usage using the [Filecoin Network Indexer](https://docs.cid.contact/filecoin-network-indexer/overview) and the DHT, making first a query to the indexer, and timing out after 3 seconds. + +``` +$ ipfs config Routing.Type --json '"custom"' + +$ ipfs config Routing.Routers.CidContact --json '{ + "Type": "reframe", + "Parameters": { + "Endpoint": "https://cid.contact/reframe" + } +}' + +$ ipfs config Routing.Routers.WanDHT --json '{ + "Type": "dht", + "Parameters": { + "Mode": "auto", + "PublicIPNetwork": true, + "AcceleratedDHTClient": false + } +}' + +$ ipfs config Routing.Routers.ParallelHelper --json '{ + "Type": "parallel", + "Parameters": { + "Routers": [ + { + "RouterName" : "CidContact", + "IgnoreErrors" : true, + "Timeout": "3s" + }, + { + "RouterName" : "WanDHT", + "IgnoreErrors" : false, + "Timeout": "5m", + "ExecuteAfter": "2s" + } + ] + } +}' + +ipfs config Routing.Methods --json '{ + "find-peers": { + "RouterName": "ParallelHelper" + }, + "find-providers": { + "RouterName": "ParallelHelper" + }, + "get-ipns": { + "RouterName": "ParallelHelper" + }, + "provide": { + "RouterName": "WanDHT" + }, + "put-ipns": { + "RouterName": "ParallelHelper" + } + }' + +``` + +### Changelog + + + +### Contributors + + \ No newline at end of file diff --git a/docs/config.md b/docs/config.md index 8f46eb67b0a..fcd67e8c6c1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -105,8 +105,8 @@ config file at runtime. - [`Routing`](#routing) - [`Routing.Routers`](#routingrouters) - [`Routing.Routers: Type`](#routingrouters-type) - - [`Routing.Routers: Enabled`](#routingrouters-enabled) - [`Routing.Routers: Parameters`](#routingrouters-parameters) + - [`Routing: Methods`](#routing-methods) - [`Routing.Type`](#routingtype) - [`Swarm`](#swarm) - [`Swarm.AddrFilters`](#swarmaddrfilters) @@ -1291,20 +1291,11 @@ It specifies the routing type that will be created. Currently supported types: - `reframe` (delegated routing based on the [reframe protocol](https://github.com/ipfs/specs/tree/main/reframe#readme)) -- `dht` (WIP, custom DHT will be added in a future release) +- `dht` +- `parallel` and `sequential`: Helpers that can be used to run several routers sequentially or in parallel. Type: `string` -#### `Routing.Routers: Enabled` - -**EXPERIMENTAL: `Routing.Routers` configuration may change in future release** - -Optional flag to disable the specified router without removing it from the configuration file. - -Default: `true` - -Type: `flag` (`null`/missing will apply the default) - #### `Routing.Routers: Parameters` **EXPERIMENTAL: `Routing.Routers` configuration may change in future release** @@ -1313,7 +1304,26 @@ Parameters needed to create the specified router. Supported params per router ty Reframe: - `Endpoint` (mandatory): URL that will be used to connect to a specified router. - - `Priority` (optional): Priority is used when making a routing request. Small numbers represent more important routers. The default priority is 100000. + +DHT: + - `"Mode"`: Mode used by the DHT. Possible values: "server", "client", "auto" + - `"AcceleratedDHTClient"`: Set to `true` if you want to use the experimentalDHT. + - `"PublicIPNetwork"`: Set to `true` to create a `WAN` DHT. Set to `false` to create a `LAN` DHT. + +Parallel: + - `Routers`: A list of routers that will be executed in parallel: + - `Name:string`: Name of the router. It should be one of the previously added to `Routers` list. + - `Timeout:duration`: Local timeout. It accepts strings compatible with Go `time.ParseDuration(string)` (`10s`, `1m`, `2h`). Time will start counting when this specific router is called, and it will stop when the router returns, or we reach the specified timeout. + - `ExecuteAfter:duration`: Providing this param will delay the execution of that router at the specified time. It accepts strings compatible with Go `time.ParseDuration(string)` (`10s`, `1m`, `2h`). + - `IgnoreErrors:bool`: It will specify if that router should be ignored if an error occurred. + - `Timeout:duration`: Global timeout. It accepts strings compatible with Go `time.ParseDuration(string)` (`10s`, `1m`, `2h`). + +Sequential: + - `Routers`: A list of routers that will be executed in order: + - `Name:string`: Name of the router. It should be one of the previously added to `Routers` list. + - `Timeout:duration`: Local timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. Time will start counting when this specific router is called, and it will stop when the router returns, or we reach the specified timeout. + - `IgnoreErrors:bool`: It will specify if that router should be ignored if an error occurred. + - `Timeout:duration`: Global timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. **Examples:** @@ -1334,13 +1344,106 @@ Default: `{}` (use the safe implicit defaults) Type: `object[string->string]` +### `Routing: Methods` + +`Methods:map` will define which routers will be executed per method. The key will be the name of the method: `"provide"`, `"find-providers"`, `"find-peers"`, `"put-ipns"`, `"get-ipns"`. All methods must be added to the list. + +The value will contain: +- `RouterName:string`: Name of the router. It should be one of the previously added to `Routing.Routers` list. + +Type: `object[string->object]` + +**Examples:** + +To use the previously added `CidContact` reframe router on all methods: + +```console +$ ipfs config Routing.Methods --json '{ + "find-peers": { + "RouterName": "CidContact" + }, + "find-providers": { + "RouterName": "CidContact" + }, + "get-ipns": { + "RouterName": "CidContact" + }, + "provide": { + "RouterName": "CidContact" + }, + "put-ipns": { + "RouterName": "CidContact" + } + }' +``` +Complete example using 3 Routers, reframe, DHT and parallel. + +``` +$ ipfs config Routing.Type --json '"custom"' + +$ ipfs config Routing.Routers.CidContact --json '{ + "Type": "reframe", + "Parameters": { + "Endpoint": "https://cid.contact/reframe" + } +}' + +$ ipfs config Routing.Routers.WanDHT --json '{ + "Type": "dht", + "Parameters": { + "Mode": "auto", + "PublicIPNetwork": true, + "AcceleratedDHTClient": false + } +}' + +$ ipfs config Routing.Routers.ParallelHelper --json '{ + "Type": "parallel", + "Parameters": { + "Routers": [ + { + "RouterName" : "CidContact", + "IgnoreErrors" : true, + "Timeout": "3s" + }, + { + "RouterName" : "WanDHT", + "IgnoreErrors" : false, + "Timeout": "5m", + "ExecuteAfter": "2s" + } + ] + } +}' + +ipfs config Routing.Methods --json '{ + "find-peers": { + "RouterName": "ParallelHelper" + }, + "find-providers": { + "RouterName": "ParallelHelper" + }, + "get-ipns": { + "RouterName": "ParallelHelper" + }, + "provide": { + "RouterName": "WanDHT" + }, + "put-ipns": { + "RouterName": "ParallelHelper" + } + }' + +``` + ### `Routing.Type` -There are two core routing options: "none" and "dht" (default). +There are three core routing options: "none", "dht" (default) and "custom". * If set to "none", your node will use _no_ routing system. You'll have to explicitly connect to peers that have the content you're looking for. * If set to "dht" (or "dhtclient"/"dhtserver"), your node will use the IPFS DHT. +* If set to "custom", `Routing.Routers` will be used. When the DHT is enabled, it can operate in two modes: client and server. diff --git a/docs/delegated-routing.md b/docs/delegated-routing.md new file mode 100644 index 00000000000..9e2b9c48b2c --- /dev/null +++ b/docs/delegated-routing.md @@ -0,0 +1,462 @@ +# New multi-router configuration system + +- Start Date: 2022-08-15 +- Related Issues: + - https://github.com/ipfs/kubo/issues/9188 + - https://github.com/ipfs/kubo/issues/9079 + +## Summary + +Previously we only used DHT for content routing and content providing. After kubo-0.14.0 release we added support for [delegated routing using Reframe protocol](https://github.com/ipfs/kubo/pull/8997). + +Now we need a better way to add different routers using different protocols like Reframe or DHT, and be able to configure them to cover different use cases. + +## Motivation + +The actual routing implementation is not enough. Some users needs to have more options when configuring the routing system. The new implementations should be able to: + +- [x] Be user-friendly and easy enough to configure, but also versatile +- [x] Configurable Router execution order + - [x] Delay some of the Router methods execution when they will be executed on parallel +- [x] Configure which method of a giving router will be used +- [x] Mark some router methods as mandatory to make the execution fails if that method fails + +## Detailed design + +### Configuration file description + +The `Routing` configuration section will contain the following keys: + +#### Type + +`Type` will be still in use to avoid complexity for the user that only wants to use Kubo with the default behavior. We are going to add a new type, `custom`, that will use the new router systems. `none` type will deactivate **all** routers, default dht and delegated ones. + +#### Routers + +`Routers` will be a key-value list of routers that will be available to use. The key is the router name and the value is all the needed configurations for that router. the `Type` will define the routing kind. The main router types will be `reframe` and `dht`, but we will implement two special routers used to execute a set of routers in parallel or sequentially: `parallel` router and `sequential` router. + +Depending on the routing type, it will use different parameters: + +##### Reframe + +Params: + +- `"Endpoint"`: URL endpoint implementing Reframe protocol. + +##### DHT + +Params: +- `"Mode"`: Mode used by the DHT. Possible values: "server", "client", "auto" +- `"AcceleratedDHTClient"`: Set to `true` if you want to use the experimentalDHT. +- `"PublicIPNetwork"`: Set to `true` to create a `WAN` DHT. Set to `false` to create a `LAN` DHT. + +##### Parallel + +Params: +- `Routers`: A list of routers that will be executed in parallel: + - `Name:string`: Name of the router. It should be one of the previously added to `Routers` list. + - `Timeout:duration`: Local timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. Time will start counting when this specific router is called, and it will stop when the router returns, or we reach the specified timeout. + - `ExecuteAfter:duration`: Providing this param will delay the execution of that router at the specified time. It accepts strings compatible with Go `time.ParseDuration(string)`. + - `IgnoreErrors:bool`: It will specify if that router should be ignored if an error occurred. +- `Timeout:duration`: Global timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. +##### Sequential + +Params: +- `Routers`: A list of routers that will be executed in order: + - `Name:string`: Name of the router. It should be one of the previously added to `Routers` list. + - `Timeout:duration`: Local timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. Time will start counting when this specific router is called, and it will stop when the router returns, or we reach the specified timeout. + - `IgnoreErrors:bool`: It will specify if that router should be ignored if an error occurred. +- `Timeout:duration`: Global timeout. It accepts strings compatible with Go `time.ParseDuration(string)`. +#### Methods + +`Methods:map` will define which routers will be executed per method. The key will be the name of the method: `"provide"`, `"find-providers"`, `"find-peers"`, `"put-ipns"`, `"get-ipns"`. All methods must be added to the list. This will make configuration discoverable giving good errors to the user if a method is missing. + +The value will contain: +- `RouterName:string`: Name of the router. It should be one of the previously added to `Routers` list. + +#### Configuration file example: + +```json +"Routing": { + "Type": "custom", + "Routers": { + "storetheindex": { + "Type": "reframe", + "Parameters": { + "Endpoint": "https://cid.contact/reframe" + } + }, + "dht-lan": { + "Type": "dht", + "Parameters": { + "Mode": "server", + "PublicIPNetwork": false, + "AcceleratedDHTClient": false + } + }, + "dht-wan": { + "Type": "dht", + "Parameters": { + "Mode": "auto", + "PublicIPNetwork": true, + "AcceleratedDHTClient": false + } + }, + "find-providers-router": { + "Type": "parallel", + "Parameters": { + "Routers": [ + { + "RouterName": "dht-lan", + "IgnoreErrors": true + }, + { + "RouterName": "dht-wan" + }, + { + "RouterName": "storetheindex" + } + ] + } + }, + "provide-router": { + "Type": "parallel", + "Parameters": { + "Routers": [ + { + "RouterName": "dht-lan", + "IgnoreErrors": true + }, + { + "RouterName": "dht-wan", + "ExecuteAfter": "100ms", + "Timeout": "100ms" + }, + { + "RouterName": "storetheindex", + "ExecuteAfter": "100ms" + } + ] + } + }, + "get-ipns-router": { + "Type": "sequential", + "Parameters": { + "Routers": [ + { + "RouterName": "dht-lan", + "IgnoreErrors": true + }, + { + "RouterName": "dht-wan", + "Timeout": "300ms" + }, + { + "RouterName": "storetheindex", + "Timeout": "300ms" + } + ] + } + }, + "put-ipns-router": { + "Type": "parallel", + "Parameters": { + "Routers": [ + { + "RouterName": "dht-lan" + }, + { + "RouterName": "dht-wan" + }, + { + "RouterName": "storetheindex" + } + ] + } + } + }, + "Methods": { + "find-providers": { + "RouterName": "find-providers-router" + }, + "provide": { + "RouterName": "provide-router" + }, + "get-ipns": { + "RouterName": "get-ipns-router" + }, + "put-ipns": { + "RouterName": "put-ipns-router" + } + } +} +``` + +Added YAML for clarity: + +```yaml +--- +Type: custom +Routers: + storetheindex: + Type: reframe + Parameters: + Endpoint: https://cid.contact/reframe + dht-lan: + Type: dht + Parameters: + Mode: server + PublicIPNetwork: false + AcceleratedDHTClient: false + dht-wan: + Type: dht + Parameters: + Mode: auto + PublicIPNetwork: true + AcceleratedDHTClient: false + find-providers-router: + Type: parallel + Parameters: + Routers: + - RouterName: dht-lan + IgnoreErrors: true + - RouterName: dht-wan + - RouterName: storetheindex + provide-router: + Type: parallel + Parameters: + Routers: + - RouterName: dht-lan + IgnoreErrors: true + - RouterName: dht-wan + ExecuteAfter: 100ms + Timeout: 100ms + - RouterName: storetheindex + ExecuteAfter: 100ms + get-ipns-router: + Type: sequential + Parameters: + Routers: + - RouterName: dht-lan + IgnoreErrors: true + - RouterName: dht-wan + Timeout: 300ms + - RouterName: storetheindex + Timeout: 300ms + put-ipns-router: + Type: parallel + Parameters: + Routers: + - RouterName: dht-lan + - RouterName: dht-wan + - RouterName: storetheindex +Methods: + find-providers: + RouterName: find-providers-router + provide: + RouterName: provide-router + get-ipns: + RouterName: get-ipns-router + put-ipns: + RouterName: put-ipns-router +``` + +### Error cases + - If any of the routers fails, the output will be an error by default. + - You can use `IgnoreErrors:true` to ignore errors for a specific router output + - To avoid any error at the output, you must ignore all router errors. + +### Implementation Details + +#### Methods + +All routers must implement the `routing.Routing` interface: + +```go= +type Routing interface { + ContentRouting + PeerRouting + ValueStore + + Bootstrap(context.Context) error +} +``` + +All methods involved: + +```go= +type Routing interface { + Provide(context.Context, cid.Cid, bool) error + FindProvidersAsync(context.Context, cid.Cid, int) <-chan peer.AddrInfo + + FindPeer(context.Context, peer.ID) (peer.AddrInfo, error) + + PutValue(context.Context, string, []byte, ...Option) error + GetValue(context.Context, string, ...Option) ([]byte, error) + SearchValue(context.Context, string, ...Option) (<-chan []byte, error) + + Bootstrap(context.Context) error +} +``` +We can configure which methods will be used per routing implementation. Methods names used in the configuration file will be: + +- `Provide`: `"provide"` +- `FindProvidersAsync`: `"find-providers"` +- `FindPeer`: `"find-peers"` +- `PutValue`: `"put-ipns"` +- `GetValue`, `SearchValue`: `"get-ipns"` +- `Bootstrap`: It will be always executed when needed. + +#### Routers + +We need to implement the `parallel` and `sequential` routers and stop using `routinghelpers.Tiered` router implementation. + +Add cycle detection to avoid to user some headaches. + +Also we need to implement an internal router, that will define the router used per method. + +#### Other considerations + +- We need to refactor how DHT routers are created to be able to use and add any amount of custom DHT routers. +- We need to add a new `custom` router type to be able to use the new routing system. +- Bitswap WANT broadcasting is not included on this document, but it can be added in next iterations. +- This document will live in docs/design-notes for historical reasons and future reference. + +## Test fixtures + +As test fixtures we can add different use cases here and see how the configuration will look like. + +### Mimic previous dual DHT config + +```json +"Routing": { + "Type": "custom", + "Routers": { + "dht-lan": { + "Type": "dht", + "Parameters": { + "Mode": "server", + "PublicIPNetwork": false + } + }, + "dht-wan": { + "Type": "dht", + "Parameters": { + "Mode": "auto", + "PublicIPNetwork": true + } + }, + "parallel-dht-strict": { + "Type": "parallel", + "Parameters": { + "Routers": [ + { + "RouterName": "dht-lan" + }, + { + "RouterName": "dht-wan" + } + ] + } + }, + "parallel-dht": { + "Type": "parallel", + "Parameters": { + "Routers": [ + { + "RouterName": "dht-lan", + "IgnoreError": true + }, + { + "RouterName": "dht-wan" + } + ] + } + } + }, + "Methods": { + "provide": { + "RouterName": "dht-wan" + }, + "find-providers": { + "RouterName": "parallel-dht-strict" + }, + "find-peers": { + "RouterName": "parallel-dht-strict" + }, + "get-ipns": { + "RouterName": "parallel-dht" + }, + "put-ipns": { + "RouterName": "parallel-dht" + } + } +} +``` + +Yaml representation for clarity: + +```yaml +--- +Type: custom +Routers: + dht-lan: + Type: dht + Parameters: + Mode: server + PublicIPNetwork: false + dht-wan: + Type: dht + Parameters: + Mode: auto + PublicIPNetwork: true + parallel-dht-strict: + Type: parallel + Parameters: + Routers: + - RouterName: dht-lan + - RouterName: dht-wan + parallel-dht: + Type: parallel + Parameters: + Routers: + - RouterName: dht-lan + IgnoreError: true + - RouterName: dht-wan +Methods: + provide: + RouterName: dht-wan + find-providers: + RouterName: parallel-dht-strict + find-peers: + RouterName: parallel-dht-strict + get-ipns: + RouterName: parallel-dht + put-ipns: + RouterName: parallel-dht + +``` + +### Compatibility + +~~We need to create a config migration using [fs-repo-migrations](https://github.com/ipfs/fs-repo-migrations). We should remove the `Routing.Type` param and add the configuration specified [previously](#Mimic-previous-dual-DHT-config).~~ + +We don't need to create any config migration! To avoid to the users the hassle of understanding how the new routing system works, we are gonna keep the old behavior. We will add the Type `custom` to make available the new Routing system. + +### Security + +No new security implications or considerations were found. + +### Alternatives + +I got ideas from all of the following links to create this design document: + +- https://github.com/ipfs/kubo/issues/9079#issuecomment-1211288268 +- https://github.com/ipfs/kubo/issues/9157 +- https://github.com/ipfs/kubo/issues/9079#issuecomment-1205000253 +- https://www.notion.so/pl-strflt/Delegated-Routing-Thoughts-very-very-WIP-0543bc51b1bd4d63a061b0f28e195d38 +- https://gist.github.com/guseggert/effa027ff4cbadd7f67598efb6704d12 + +### Copyright + +Copyright and related rights waived via [CC0](https://creativecommons.org/publicdomain/zero/1.0/). diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index f94b4e7e53d..549d59afa01 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -68,7 +68,7 @@ require ( github.com/ipfs/go-cid v0.3.2 // indirect github.com/ipfs/go-cidutil v0.1.0 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect - github.com/ipfs/go-delegated-routing v0.3.0 // indirect + github.com/ipfs/go-delegated-routing v0.6.0 // indirect github.com/ipfs/go-ds-badger v0.3.0 // indirect github.com/ipfs/go-ds-flatfs v0.5.1 // indirect github.com/ipfs/go-ds-leveldb v0.5.0 // indirect @@ -106,9 +106,9 @@ require ( github.com/ipfs/go-unixfs v0.4.0 // indirect github.com/ipfs/go-unixfsnode v1.4.0 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect - github.com/ipld/edelweiss v0.1.4 // indirect + github.com/ipld/edelweiss v0.2.0 // indirect github.com/ipld/go-codec-dagpb v1.4.1 // indirect - github.com/ipld/go-ipld-prime v0.17.0 // indirect + github.com/ipld/go-ipld-prime v0.18.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -127,7 +127,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.6.1 // indirect github.com/libp2p/go-libp2p-pubsub-router v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-libp2p-routing-helpers v0.2.3 // indirect + github.com/libp2p/go-libp2p-routing-helpers v0.4.0 // indirect github.com/libp2p/go-libp2p-xor v0.1.0 // indirect github.com/libp2p/go-mplex v0.7.0 // indirect github.com/libp2p/go-msgio v0.2.0 // indirect @@ -209,7 +209,7 @@ require ( golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect + golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.12 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index b8800b88c1a..30ccf800f64 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -477,6 +477,7 @@ github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqg github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-cid v0.3.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-cidutil v0.0.2/go.mod h1:ewllrvrxG6AMYStla3GD7Cqn+XYSLqjK0vc+086tB6s= @@ -495,8 +496,8 @@ github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh7 github.com/ipfs/go-datastore v0.5.1/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= -github.com/ipfs/go-delegated-routing v0.3.0 h1:pF5apOJ/xdQkj22mRahW9GmSuCkgMLparKZWKJBO4CE= -github.com/ipfs/go-delegated-routing v0.3.0/go.mod h1:2w79E1/G9YOaxyJJQgqIFSQaa/GdS2zSATEpK8aJUBM= +github.com/ipfs/go-delegated-routing v0.6.0 h1:+M1siyTB2H4mHzEnbWjepQxlmKbapVWdbYLexSDODpg= +github.com/ipfs/go-delegated-routing v0.6.0/go.mod h1:FJjhCChfcWK9z6OXo2jwKKJoxq1JlEWG7YTvwaA7UbI= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -653,8 +654,8 @@ github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0 github.com/ipfs/interface-go-ipfs-core v0.7.0 h1:7tb+2upz8oCcjIyjo1atdMk+P+u7wPmI+GksBlLE8js= github.com/ipfs/interface-go-ipfs-core v0.7.0/go.mod h1:lF27E/nnSPbylPqKVXGZghal2hzifs3MmjyiEjnc9FY= github.com/ipfs/tar-utils v0.0.2/go.mod h1:4qlnRWgTVljIMhSG2SqRYn66NT+3wrv/kZt9V+eqxDM= -github.com/ipld/edelweiss v0.1.4 h1:g4+C2Ph+8SV2MCJBG3oRtetvxJYAS2WzlNGgsOY95iM= -github.com/ipld/edelweiss v0.1.4/go.mod h1:JX1MR06BPcTOF+5xCYDLnylYkXS15iUN0/RXVSiUIQs= +github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk= +github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4= github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ= github.com/ipld/go-car v0.4.0/go.mod h1:Uslcn4O9cBKK9wqHm/cLTFacg6RAPv6LZx2mxd2Ypl4= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= @@ -670,8 +671,9 @@ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHt github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= -github.com/ipld/go-ipld-prime v0.17.0 h1:+U2peiA3aQsE7mrXjD2nYZaZrCcakoz2Wge8K42Ld8g= github.com/ipld/go-ipld-prime v0.17.0/go.mod h1:aYcKm5TIvGfY8P3QBKz/2gKcLxzJ1zDaD+o0bOowhgs= +github.com/ipld/go-ipld-prime v0.18.0 h1:xUk7NUBSWHEXdjiOu2sLXouFJOMs0yoYzeI5RAqhYQo= +github.com/ipld/go-ipld-prime v0.18.0/go.mod h1:735yXW548CKrLwVCYXzqx90p5deRJMVVxM9eJ4Qe+qE= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= @@ -945,8 +947,9 @@ github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvN github.com/libp2p/go-libp2p-resource-manager v0.1.5/go.mod h1:wJPNjeE4XQlxeidwqVY5G6DLOKqFK33u2n8blpl0I6Y= github.com/libp2p/go-libp2p-resource-manager v0.3.0/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= -github.com/libp2p/go-libp2p-routing-helpers v0.2.3 h1:xY61alxJ6PurSi+MXbywZpelvuU4U4p/gPTxjqCqTzY= github.com/libp2p/go-libp2p-routing-helpers v0.2.3/go.mod h1:795bh+9YeoFl99rMASoiVgHdi5bjack0N1+AFAdbvBw= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0 h1:b7y4aixQ7AwbqYfcOQ6wTw8DQvuRZeTAA0Od3YYN5yc= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E= github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= @@ -1286,6 +1289,7 @@ github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.0.16/go.mod h1:zhfEIgVnB/rPMfxgFw15ZmGoNaKyNUIE4IWHG/kC+Ag= github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= +github.com/multiformats/go-multihash v0.2.0/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= @@ -1947,8 +1951,9 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 h1:ohgcoMbSofXygzo6AD2I1kz3BFmW1QArPYTtwEM3UXc= +golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/go.mod b/go.mod index df84a21ca38..38f407cc862 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( github.com/ipld/go-car v0.4.0 github.com/ipld/go-car/v2 v2.4.0 github.com/ipld/go-codec-dagpb v1.4.1 - github.com/ipld/go-ipld-prime v0.17.0 + github.com/ipld/go-ipld-prime v0.18.0 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 github.com/jbenet/goprocess v0.1.4 @@ -77,7 +77,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.6.1 github.com/libp2p/go-libp2p-pubsub-router v0.5.0 github.com/libp2p/go-libp2p-record v0.2.0 - github.com/libp2p/go-libp2p-routing-helpers v0.2.3 + github.com/libp2p/go-libp2p-routing-helpers v0.4.0 github.com/libp2p/go-libp2p-testing v0.12.0 github.com/libp2p/go-socket-activation v0.1.0 github.com/miekg/dns v1.1.50 @@ -111,12 +111,12 @@ require ( go.uber.org/zap v1.23.0 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab + golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 ) require ( github.com/benbjohnson/clock v1.3.0 - github.com/ipfs/go-delegated-routing v0.3.0 + github.com/ipfs/go-delegated-routing v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 ) @@ -167,7 +167,7 @@ require ( github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.2 // indirect github.com/ipfs/go-peertaskqueue v0.7.1 // indirect - github.com/ipld/edelweiss v0.1.4 // indirect + github.com/ipld/edelweiss v0.2.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/klauspost/compress v1.15.10 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect diff --git a/go.sum b/go.sum index 65b60cc53c9..734e5ea1edb 100644 --- a/go.sum +++ b/go.sum @@ -489,8 +489,8 @@ github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= -github.com/ipfs/go-delegated-routing v0.3.0 h1:pF5apOJ/xdQkj22mRahW9GmSuCkgMLparKZWKJBO4CE= -github.com/ipfs/go-delegated-routing v0.3.0/go.mod h1:2w79E1/G9YOaxyJJQgqIFSQaa/GdS2zSATEpK8aJUBM= +github.com/ipfs/go-delegated-routing v0.6.0 h1:+M1siyTB2H4mHzEnbWjepQxlmKbapVWdbYLexSDODpg= +github.com/ipfs/go-delegated-routing v0.6.0/go.mod h1:FJjhCChfcWK9z6OXo2jwKKJoxq1JlEWG7YTvwaA7UbI= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= @@ -649,8 +649,8 @@ github.com/ipfs/interface-go-ipfs-core v0.7.0 h1:7tb+2upz8oCcjIyjo1atdMk+P+u7wPm github.com/ipfs/interface-go-ipfs-core v0.7.0/go.mod h1:lF27E/nnSPbylPqKVXGZghal2hzifs3MmjyiEjnc9FY= github.com/ipfs/tar-utils v0.0.2 h1:UNgHB4x/PPzbMkmJi+7EqC9LNMPDztOVSnx1HAqSNg4= github.com/ipfs/tar-utils v0.0.2/go.mod h1:4qlnRWgTVljIMhSG2SqRYn66NT+3wrv/kZt9V+eqxDM= -github.com/ipld/edelweiss v0.1.4 h1:g4+C2Ph+8SV2MCJBG3oRtetvxJYAS2WzlNGgsOY95iM= -github.com/ipld/edelweiss v0.1.4/go.mod h1:JX1MR06BPcTOF+5xCYDLnylYkXS15iUN0/RXVSiUIQs= +github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk= +github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4= github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ= github.com/ipld/go-car v0.4.0/go.mod h1:Uslcn4O9cBKK9wqHm/cLTFacg6RAPv6LZx2mxd2Ypl4= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= @@ -666,8 +666,8 @@ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHt github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= -github.com/ipld/go-ipld-prime v0.17.0 h1:+U2peiA3aQsE7mrXjD2nYZaZrCcakoz2Wge8K42Ld8g= -github.com/ipld/go-ipld-prime v0.17.0/go.mod h1:aYcKm5TIvGfY8P3QBKz/2gKcLxzJ1zDaD+o0bOowhgs= +github.com/ipld/go-ipld-prime v0.18.0 h1:xUk7NUBSWHEXdjiOu2sLXouFJOMs0yoYzeI5RAqhYQo= +github.com/ipld/go-ipld-prime v0.18.0/go.mod h1:735yXW548CKrLwVCYXzqx90p5deRJMVVxM9eJ4Qe+qE= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= @@ -936,8 +936,9 @@ github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3qu github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= github.com/libp2p/go-libp2p-resource-manager v0.1.5/go.mod h1:wJPNjeE4XQlxeidwqVY5G6DLOKqFK33u2n8blpl0I6Y= github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= -github.com/libp2p/go-libp2p-routing-helpers v0.2.3 h1:xY61alxJ6PurSi+MXbywZpelvuU4U4p/gPTxjqCqTzY= github.com/libp2p/go-libp2p-routing-helpers v0.2.3/go.mod h1:795bh+9YeoFl99rMASoiVgHdi5bjack0N1+AFAdbvBw= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0 h1:b7y4aixQ7AwbqYfcOQ6wTw8DQvuRZeTAA0Od3YYN5yc= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E= github.com/libp2p/go-libp2p-secio v0.0.3/go.mod h1:hS7HQ00MgLhRO/Wyu1bTX6ctJKhVpm+j2/S2A5UqYb0= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= @@ -1907,8 +1908,9 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41 h1:ohgcoMbSofXygzo6AD2I1kz3BFmW1QArPYTtwEM3UXc= +golang.org/x/sys v0.0.0-20220915200043-7b5979e65e41/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= diff --git a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go index ba64b96dfc0..cc10de4eba5 100644 --- a/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go +++ b/repo/fsrepo/migrations/ipfsfetcher/ipfsfetcher.go @@ -188,7 +188,7 @@ func initTempNode(ctx context.Context, bootstrap []string, peers []peer.AddrInfo } // configure the temporary node - cfg.Routing.Type = config.NewOptionalString("dhtclient") + cfg.Routing.Type = "dhtclient" // Disable listening for inbound connections cfg.Addresses.Gateway = []string{} diff --git a/routing/composer.go b/routing/composer.go new file mode 100644 index 00000000000..ed3f2d13ec6 --- /dev/null +++ b/routing/composer.go @@ -0,0 +1,74 @@ +package routing + +import ( + "context" + + "github.com/hashicorp/go-multierror" + "github.com/ipfs/go-cid" + routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" + "github.com/multiformats/go-multihash" +) + +var _ routinghelpers.ProvideManyRouter = &Composer{} +var _ routing.Routing = &Composer{} + +type Composer struct { + GetValueRouter routing.Routing + PutValueRouter routing.Routing + FindPeersRouter routing.Routing + FindProvidersRouter routing.Routing + ProvideRouter routing.Routing +} + +func (c *Composer) Provide(ctx context.Context, cid cid.Cid, provide bool) error { + return c.ProvideRouter.Provide(ctx, cid, provide) +} + +func (c *Composer) ProvideMany(ctx context.Context, keys []multihash.Multihash) error { + pmr, ok := c.ProvideRouter.(routinghelpers.ProvideManyRouter) + if !ok { + return nil + } + + return pmr.ProvideMany(ctx, keys) +} + +func (c *Composer) Ready() bool { + pmr, ok := c.ProvideRouter.(routinghelpers.ProvideManyRouter) + if !ok { + return false + } + + return pmr.Ready() +} + +func (c *Composer) FindProvidersAsync(ctx context.Context, cid cid.Cid, count int) <-chan peer.AddrInfo { + return c.FindProvidersRouter.FindProvidersAsync(ctx, cid, count) +} + +func (c *Composer) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInfo, error) { + return c.FindPeersRouter.FindPeer(ctx, pid) +} + +func (c *Composer) PutValue(ctx context.Context, key string, val []byte, opts ...routing.Option) error { + return c.PutValueRouter.PutValue(ctx, key, val, opts...) +} + +func (c *Composer) GetValue(ctx context.Context, key string, opts ...routing.Option) ([]byte, error) { + return c.GetValueRouter.GetValue(ctx, key, opts...) +} + +func (c *Composer) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) { + return c.GetValueRouter.SearchValue(ctx, key, opts...) +} + +func (c *Composer) Bootstrap(ctx context.Context) error { + errfp := c.FindPeersRouter.Bootstrap(ctx) + errfps := c.FindProvidersRouter.Bootstrap(ctx) + errgv := c.GetValueRouter.Bootstrap(ctx) + errpv := c.PutValueRouter.Bootstrap(ctx) + errp := c.ProvideRouter.Bootstrap(ctx) + return multierror.Append(errfp, errfps, errgv, errpv, errp) +} diff --git a/routing/delegated.go b/routing/delegated.go index 030fae8a3f6..d8e818bc3c2 100644 --- a/routing/delegated.go +++ b/routing/delegated.go @@ -1,93 +1,315 @@ package routing import ( - "strconv" + "context" + "encoding/base64" + "errors" + "fmt" + "net/http" + "github.com/ipfs/go-datastore" drc "github.com/ipfs/go-delegated-routing/client" drp "github.com/ipfs/go-delegated-routing/gen/proto" + logging "github.com/ipfs/go-log" "github.com/ipfs/kubo/config" + dht "github.com/libp2p/go-libp2p-kad-dht" + "github.com/libp2p/go-libp2p-kad-dht/dual" + "github.com/libp2p/go-libp2p-kad-dht/fullrt" + record "github.com/libp2p/go-libp2p-record" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" + ic "github.com/libp2p/go-libp2p/core/crypto" + host "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" + ma "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multicodec" ) -type TieredRouter interface { - routing.Routing - ProvideMany() ProvideMany -} +var log = logging.Logger("routing/delegated") -var _ TieredRouter = &Tiered{} +func Parse(routers config.Routers, methods config.Methods, extraDHT *ExtraDHTParams, extraReframe *ExtraReframeParams) (routing.Routing, error) { + if err := methods.Check(); err != nil { + return nil, err + } -// Tiered is a routing Tiered implementation providing some extra methods to fill -// some special use cases when initializing the client. -type Tiered struct { - routinghelpers.Tiered -} + createdRouters := make(map[string]routing.Routing) + finalRouter := &Composer{} -// ProvideMany returns a ProvideMany implementation including all Routers that -// implements ProvideMany -func (ds Tiered) ProvideMany() ProvideMany { - var pms []ProvideMany - for _, r := range ds.Tiered.Routers { - pm, ok := r.(ProvideMany) - if !ok { - continue + // Create all needed routers from method names + for mn, m := range methods { + router, err := parse(make(map[string]bool), createdRouters, m.RouterName, routers, extraDHT, extraReframe) + if err != nil { + return nil, err } - pms = append(pms, pm) - } - if len(pms) == 0 { - return nil + switch mn { + case config.MethodNamePutIPNS: + finalRouter.PutValueRouter = router + case config.MethodNameGetIPNS: + finalRouter.GetValueRouter = router + case config.MethodNameFindPeers: + finalRouter.FindPeersRouter = router + case config.MethodNameFindProviders: + finalRouter.FindProvidersRouter = router + case config.MethodNameProvide: + finalRouter.ProvideRouter = router + } + + log.Info("using method ", mn, " with router ", m.RouterName) } - return &ProvideManyWrapper{pms: pms} + return finalRouter, nil } -const defaultPriority = 100000 +func parse(visited map[string]bool, + createdRouters map[string]routing.Routing, + routerName string, + routersCfg config.Routers, + extraDHT *ExtraDHTParams, + extraReframe *ExtraReframeParams, +) (routing.Routing, error) { + // check if we already created it + r, ok := createdRouters[routerName] + if ok { + return r, nil + } -// GetPriority extract priority from config params. -// Small numbers represent more important routers. -func GetPriority(params map[string]string) int { - param := params[string(config.RouterParamPriority)] - if param == "" { - return defaultPriority + // check if we are in a dep loop + if visited[routerName] { + return nil, fmt.Errorf("dependency loop creating router with name %q", routerName) } - p, err := strconv.Atoi(param) - if err != nil { - return defaultPriority + // set node as visited + visited[routerName] = true + + cfg, ok := routersCfg[routerName] + if !ok { + return nil, fmt.Errorf("config for router with name %q not found", routerName) } - return p -} + var router routing.Routing + var err error + switch cfg.Type { + case config.RouterTypeReframe: + router, err = reframeRoutingFromConfig(cfg.Router, extraReframe) + case config.RouterTypeDHT: + router, err = dhtRoutingFromConfig(cfg.Router, extraDHT) + case config.RouterTypeParallel: + crp := cfg.Parameters.(*config.ComposableRouterParams) + var pr []*routinghelpers.ParallelRouter + for _, cr := range crp.Routers { + ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraReframe) + if err != nil { + return nil, err + } + + pr = append(pr, &routinghelpers.ParallelRouter{ + Router: ri, + IgnoreError: cr.IgnoreErrors, + Timeout: cr.Timeout.Duration, + ExecuteAfter: cr.ExecuteAfter.WithDefault(0), + }) + + } -// RoutingFromConfig creates a Routing instance from the specified configuration. -func RoutingFromConfig(c config.Router) (routing.Routing, error) { - switch { - case c.Type == string(config.RouterTypeReframe): - return reframeRoutingFromConfig(c) + router = routinghelpers.NewComposableParallel(pr) + case config.RouterTypeSequential: + crp := cfg.Parameters.(*config.ComposableRouterParams) + var sr []*routinghelpers.SequentialRouter + for _, cr := range crp.Routers { + ri, err := parse(visited, createdRouters, cr.RouterName, routersCfg, extraDHT, extraReframe) + if err != nil { + return nil, err + } + + sr = append(sr, &routinghelpers.SequentialRouter{ + Router: ri, + IgnoreError: cr.IgnoreErrors, + Timeout: cr.Timeout.Duration, + }) + + } + + router = routinghelpers.NewComposableSequential(sr) default: - return nil, &RouterTypeNotFoundError{c.Type} + return nil, fmt.Errorf("unknown router type %q", cfg.Type) } + + if err != nil { + return nil, err + } + + createdRouters[routerName] = router + + log.Info("created router ", routerName, " with params ", cfg.Parameters) + + return router, nil +} + +type ExtraReframeParams struct { + PeerID string + Addrs []string + PrivKeyB64 string } -func reframeRoutingFromConfig(conf config.Router) (routing.Routing, error) { +func reframeRoutingFromConfig(conf config.Router, extraReframe *ExtraReframeParams) (routing.Routing, error) { var dr drp.DelegatedRouting_Client - param := string(config.RouterParamEndpoint) - addr, ok := conf.Parameters[param] - if !ok { - return nil, NewParamNeededErr(param, conf.Type) + params := conf.Parameters.(*config.ReframeRouterParams) + + if params.Endpoint == "" { + return nil, NewParamNeededErr("Endpoint", conf.Type) } - dr, err := drp.New_DelegatedRouting_Client(addr) + // Increase per-host connection pool since we are making lots of concurrent requests. + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.MaxIdleConns = 500 + transport.MaxIdleConnsPerHost = 100 + + delegateHTTPClient := &http.Client{ + Transport: transport, + } + dr, err := drp.New_DelegatedRouting_Client(params.Endpoint, + drp.DelegatedRouting_Client_WithHTTPClient(delegateHTTPClient), + ) if err != nil { return nil, err } - c := drc.NewClient(dr) + var c *drc.Client + + // this path is for tests only + if extraReframe == nil { + c, err = drc.NewClient(dr, nil, nil) + if err != nil { + return nil, err + } + } else { + prov, err := createProvider(extraReframe.PeerID, extraReframe.Addrs) + if err != nil { + return nil, err + } + + key, err := decodePrivKey(extraReframe.PrivKeyB64) + if err != nil { + return nil, err + } + + c, err = drc.NewClient(dr, prov, key) + if err != nil { + return nil, err + } + } + crc := drc.NewContentRoutingClient(c) return &reframeRoutingWrapper{ Client: c, ContentRoutingClient: crc, }, nil } + +func decodePrivKey(keyB64 string) (ic.PrivKey, error) { + pk, err := base64.StdEncoding.DecodeString(keyB64) + if err != nil { + return nil, err + } + + return ic.UnmarshalPrivateKey(pk) +} + +func createProvider(peerID string, addrs []string) (*drc.Provider, error) { + pID, err := peer.Decode(peerID) + if err != nil { + return nil, err + } + + var mas []ma.Multiaddr + for _, a := range addrs { + m, err := ma.NewMultiaddr(a) + if err != nil { + return nil, err + } + + mas = append(mas, m) + } + + return &drc.Provider{ + Peer: peer.AddrInfo{ + ID: pID, + Addrs: mas, + }, + ProviderProto: []drc.TransferProtocol{ + {Codec: multicodec.TransportBitswap}, + }, + }, nil +} + +type ExtraDHTParams struct { + BootstrapPeers []peer.AddrInfo + Host host.Host + Validator record.Validator + Datastore datastore.Batching + Context context.Context +} + +func dhtRoutingFromConfig(conf config.Router, extra *ExtraDHTParams) (routing.Routing, error) { + params, ok := conf.Parameters.(*config.DHTRouterParams) + if !ok { + return nil, errors.New("incorrect params for DHT router") + } + + if params.AcceleratedDHTClient { + return createFullRT(extra) + } + + var mode dht.ModeOpt + switch params.Mode { + case config.DHTModeAuto: + mode = dht.ModeAuto + case config.DHTModeClient: + mode = dht.ModeClient + case config.DHTModeServer: + mode = dht.ModeServer + default: + return nil, fmt.Errorf("invalid DHT mode: %q", params.Mode) + } + + return createDHT(extra, params.PublicIPNetwork, mode) +} + +func createDHT(params *ExtraDHTParams, public bool, mode dht.ModeOpt) (routing.Routing, error) { + var opts []dht.Option + + if public { + opts = append(opts, dht.QueryFilter(dht.PublicQueryFilter), + dht.RoutingTableFilter(dht.PublicRoutingTableFilter), + dht.RoutingTablePeerDiversityFilter(dht.NewRTPeerDiversityFilter(params.Host, 2, 3))) + } else { + opts = append(opts, dht.ProtocolExtension(dual.LanExtension), + dht.QueryFilter(dht.PrivateQueryFilter), + dht.RoutingTableFilter(dht.PrivateRoutingTableFilter)) + } + + opts = append(opts, + dht.Concurrency(10), + dht.Mode(mode), + dht.Datastore(params.Datastore), + dht.Validator(params.Validator), + dht.BootstrapPeers(params.BootstrapPeers...)) + + return dht.New( + params.Context, params.Host, opts..., + ) +} + +func createFullRT(params *ExtraDHTParams) (routing.Routing, error) { + return fullrt.NewFullRT(params.Host, + dht.DefaultPrefix, + fullrt.DHTOption( + dht.Validator(params.Validator), + dht.Datastore(params.Datastore), + dht.BootstrapPeers(params.BootstrapPeers...), + dht.BucketSize(20), + ), + ) +} diff --git a/routing/delegated_test.go b/routing/delegated_test.go index c1061f0eb91..04b8e282fba 100644 --- a/routing/delegated_test.go +++ b/routing/delegated_test.go @@ -1,121 +1,239 @@ package routing import ( - "context" + "encoding/base64" "testing" - "github.com/ipfs/go-cid" "github.com/ipfs/kubo/config" - routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/routing" - "github.com/multiformats/go-multihash" + crypto "github.com/libp2p/go-libp2p/core/crypto" + peer "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" ) -func TestPriority(t *testing.T) { +func TestReframeRoutingFromConfig(t *testing.T) { require := require.New(t) - params := make(map[string]string) - p := GetPriority(params) - require.Equal(defaultPriority, p) + r, err := reframeRoutingFromConfig(config.Router{ + Type: config.RouterTypeReframe, + Parameters: &config.ReframeRouterParams{}, + }, nil) - params[string(config.RouterParamPriority)] = "101" - - p = GetPriority(params) - - require.Equal(101, p) - - params[string(config.RouterParamPriority)] = "NAN" - - p = GetPriority(params) - - require.Equal(defaultPriority, p) -} + require.Nil(r) + require.EqualError(err, "configuration param 'Endpoint' is needed for reframe delegated routing types") -func TestRoutingFromConfig(t *testing.T) { - require := require.New(t) + r, err = reframeRoutingFromConfig(config.Router{ + Type: config.RouterTypeReframe, + Parameters: &config.ReframeRouterParams{ + Endpoint: "test", + }, + }, nil) - r, err := RoutingFromConfig(config.Router{ - Type: "unknown", - }) + require.NoError(err) + require.NotNil(r) - require.Nil(r) - require.EqualError(err, "router type unknown is not supported") + priv, pub, err := crypto.GenerateKeyPair(crypto.RSA, 2048) + require.NoError(err) - r, err = RoutingFromConfig(config.Router{ - Type: string(config.RouterTypeReframe), - Parameters: make(map[string]string), - }) + id, err := peer.IDFromPublicKey(pub) + require.NoError(err) - require.Nil(r) - require.EqualError(err, "configuration param 'Endpoint' is needed for reframe delegated routing types") + privM, err := crypto.MarshalPrivateKey(priv) + require.NoError(err) - r, err = RoutingFromConfig(config.Router{ - Type: string(config.RouterTypeReframe), - Parameters: map[string]string{ - string(config.RouterParamEndpoint): "test", + r, err = reframeRoutingFromConfig(config.Router{ + Type: config.RouterTypeReframe, + Parameters: &config.ReframeRouterParams{ + Endpoint: "test", }, + }, &ExtraReframeParams{ + PeerID: id.String(), + Addrs: []string{"/ip4/0.0.0.0/tcp/4001"}, + PrivKeyB64: base64.StdEncoding.EncodeToString(privM), }) require.NotNil(r) require.NoError(err) } -func TestTieredRouter(t *testing.T) { +func TestParser(t *testing.T) { require := require.New(t) - tr := &Tiered{ - Tiered: routinghelpers.Tiered{ - Routers: []routing.Routing{routinghelpers.Null{}}, + router, err := Parse(config.Routers{ + "r1": config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeReframe, + Parameters: &config.ReframeRouterParams{ + Endpoint: "testEndpoint", + }, + }, }, - } - - pm := tr.ProvideMany() - require.Nil(pm) - - tr.Tiered.Routers = append(tr.Tiered.Routers, &dummyRouter{}) - - pm = tr.ProvideMany() - require.NotNil(pm) -} + "r2": config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeSequential, + Parameters: &config.ComposableRouterParams{ + Routers: []config.ConfigRouter{ + { + RouterName: "r1", + }, + }, + }, + }, + }, + }, config.Methods{ + config.MethodNameFindPeers: config.Method{ + RouterName: "r1", + }, + config.MethodNameFindProviders: config.Method{ + RouterName: "r1", + }, + config.MethodNameGetIPNS: config.Method{ + RouterName: "r1", + }, + config.MethodNamePutIPNS: config.Method{ + RouterName: "r2", + }, + config.MethodNameProvide: config.Method{ + RouterName: "r2", + }, + }, &ExtraDHTParams{}, nil) -type dummyRouter struct { -} + require.NoError(err) -func (dr *dummyRouter) Provide(context.Context, cid.Cid, bool) error { - panic("not implemented") + comp, ok := router.(*Composer) + require.True(ok) + require.Equal(comp.FindPeersRouter, comp.FindProvidersRouter) + require.Equal(comp.ProvideRouter, comp.PutValueRouter) } -func (dr *dummyRouter) FindProvidersAsync(context.Context, cid.Cid, int) <-chan peer.AddrInfo { - panic("not implemented") -} +func TestParserRecursive(t *testing.T) { + require := require.New(t) -func (dr *dummyRouter) FindPeer(context.Context, peer.ID) (peer.AddrInfo, error) { - panic("not implemented") -} + router, err := Parse(config.Routers{ + "reframe1": config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeReframe, + Parameters: &config.ReframeRouterParams{ + Endpoint: "testEndpoint1", + }, + }, + }, + "reframe2": config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeReframe, + Parameters: &config.ReframeRouterParams{ + Endpoint: "testEndpoint2", + }, + }, + }, + "reframe3": config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeReframe, + Parameters: &config.ReframeRouterParams{ + Endpoint: "testEndpoint3", + }, + }, + }, + "composable1": config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeSequential, + Parameters: &config.ComposableRouterParams{ + Routers: []config.ConfigRouter{ + { + RouterName: "reframe1", + }, + { + RouterName: "reframe2", + }, + }, + }, + }, + }, + "composable2": config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeParallel, + Parameters: &config.ComposableRouterParams{ + Routers: []config.ConfigRouter{ + { + RouterName: "composable1", + }, + { + RouterName: "reframe3", + }, + }, + }, + }, + }, + }, config.Methods{ + config.MethodNameFindPeers: config.Method{ + RouterName: "composable2", + }, + config.MethodNameFindProviders: config.Method{ + RouterName: "composable2", + }, + config.MethodNameGetIPNS: config.Method{ + RouterName: "composable2", + }, + config.MethodNamePutIPNS: config.Method{ + RouterName: "composable2", + }, + config.MethodNameProvide: config.Method{ + RouterName: "composable2", + }, + }, &ExtraDHTParams{}, nil) -func (dr *dummyRouter) PutValue(context.Context, string, []byte, ...routing.Option) error { - panic("not implemented") -} + require.NoError(err) -func (dr *dummyRouter) GetValue(context.Context, string, ...routing.Option) ([]byte, error) { - panic("not implemented") -} + _, ok := router.(*Composer) + require.True(ok) -func (dr *dummyRouter) SearchValue(context.Context, string, ...routing.Option) (<-chan []byte, error) { - panic("not implemented") } -func (dr *dummyRouter) Bootstrap(context.Context) error { - panic("not implemented") -} +func TestParserRecursiveLoop(t *testing.T) { + require := require.New(t) -func (dr *dummyRouter) ProvideMany(ctx context.Context, keys []multihash.Multihash) error { - panic("not implemented") -} + _, err := Parse(config.Routers{ + "composable1": config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeSequential, + Parameters: &config.ComposableRouterParams{ + Routers: []config.ConfigRouter{ + { + RouterName: "composable2", + }, + }, + }, + }, + }, + "composable2": config.RouterParser{ + Router: config.Router{ + Type: config.RouterTypeParallel, + Parameters: &config.ComposableRouterParams{ + Routers: []config.ConfigRouter{ + { + RouterName: "composable1", + }, + }, + }, + }, + }, + }, config.Methods{ + config.MethodNameFindPeers: config.Method{ + RouterName: "composable2", + }, + config.MethodNameFindProviders: config.Method{ + RouterName: "composable2", + }, + config.MethodNameGetIPNS: config.Method{ + RouterName: "composable2", + }, + config.MethodNamePutIPNS: config.Method{ + RouterName: "composable2", + }, + config.MethodNameProvide: config.Method{ + RouterName: "composable2", + }, + }, &ExtraDHTParams{}, nil) -func (dr *dummyRouter) Ready() bool { - panic("not implemented") + require.ErrorContains(err, "dependency loop creating router with name \"composable2\"") } diff --git a/routing/error.go b/routing/error.go index 15016a0e4e6..07186ab1968 100644 --- a/routing/error.go +++ b/routing/error.go @@ -1,13 +1,17 @@ package routing -import "fmt" +import ( + "fmt" + + "github.com/ipfs/kubo/config" +) type ParamNeededError struct { ParamName string - RouterType string + RouterType config.RouterType } -func NewParamNeededErr(param, routing string) error { +func NewParamNeededErr(param string, routing config.RouterType) error { return &ParamNeededError{ ParamName: param, RouterType: routing, @@ -17,11 +21,3 @@ func NewParamNeededErr(param, routing string) error { func (e *ParamNeededError) Error() string { return fmt.Sprintf("configuration param '%v' is needed for %v delegated routing types", e.ParamName, e.RouterType) } - -type RouterTypeNotFoundError struct { - RouterType string -} - -func (e *RouterTypeNotFoundError) Error() string { - return fmt.Sprintf("router type %v is not supported", e.RouterType) -} diff --git a/routing/wrapper.go b/routing/wrapper.go index 95b7cffdae4..3a64d6a6373 100644 --- a/routing/wrapper.go +++ b/routing/wrapper.go @@ -5,13 +5,13 @@ import ( "github.com/ipfs/go-cid" drc "github.com/ipfs/go-delegated-routing/client" + routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" - "github.com/multiformats/go-multihash" - "golang.org/x/sync/errgroup" ) var _ routing.Routing = &reframeRoutingWrapper{} +var _ routinghelpers.ProvideManyRouter = &reframeRoutingWrapper{} // reframeRoutingWrapper is a wrapper needed to construct the routing.Routing interface from // delegated-routing library. @@ -20,6 +20,10 @@ type reframeRoutingWrapper struct { *drc.ContentRoutingClient } +func (c *reframeRoutingWrapper) Provide(ctx context.Context, id cid.Cid, announce bool) error { + return c.ContentRoutingClient.Provide(ctx, id, announce) +} + func (c *reframeRoutingWrapper) FindProvidersAsync(ctx context.Context, cid cid.Cid, count int) <-chan peer.AddrInfo { return c.ContentRoutingClient.FindProvidersAsync(ctx, cid, count) } @@ -32,35 +36,7 @@ func (c *reframeRoutingWrapper) FindPeer(ctx context.Context, id peer.ID) (peer. return peer.AddrInfo{}, routing.ErrNotSupported } -type ProvideMany interface { - ProvideMany(ctx context.Context, keys []multihash.Multihash) error - Ready() bool -} - -var _ ProvideMany = &ProvideManyWrapper{} - -type ProvideManyWrapper struct { - pms []ProvideMany -} - -func (pmw *ProvideManyWrapper) ProvideMany(ctx context.Context, keys []multihash.Multihash) error { - var g errgroup.Group - for _, pm := range pmw.pms { - pm := pm - g.Go(func() error { - return pm.ProvideMany(ctx, keys) - }) - } - - return g.Wait() -} - -// Ready is ready if all providers are ready -func (pmw *ProvideManyWrapper) Ready() bool { - out := true - for _, pm := range pmw.pms { - out = out && pm.Ready() - } - - return out +type ProvideManyRouter interface { + routinghelpers.ProvideManyRouter + routing.Routing } diff --git a/routing/wrapper_test.go b/routing/wrapper_test.go deleted file mode 100644 index dd5f2f44690..00000000000 --- a/routing/wrapper_test.go +++ /dev/null @@ -1,101 +0,0 @@ -package routing - -import ( - "context" - "errors" - "testing" - - "github.com/multiformats/go-multihash" -) - -func TestProvideManyWrapper_ProvideMany(t *testing.T) { - type fields struct { - pms []ProvideMany - } - tests := []struct { - name string - fields fields - wantErr bool - ready bool - }{ - { - name: "one provider", - fields: fields{ - pms: []ProvideMany{ - newDummyProvideMany(true, false), - }, - }, - wantErr: false, - ready: true, - }, - { - name: "two providers, no errors and ready", - fields: fields{ - pms: []ProvideMany{ - newDummyProvideMany(true, false), - newDummyProvideMany(true, false), - }, - }, - wantErr: false, - ready: true, - }, - { - name: "two providers, no ready, no error", - fields: fields{ - pms: []ProvideMany{ - newDummyProvideMany(true, false), - newDummyProvideMany(false, false), - }, - }, - wantErr: false, - ready: false, - }, - { - name: "two providers, no ready, and one erroing", - fields: fields{ - pms: []ProvideMany{ - newDummyProvideMany(true, false), - newDummyProvideMany(false, true), - }, - }, - wantErr: true, - ready: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pmw := &ProvideManyWrapper{ - pms: tt.fields.pms, - } - if err := pmw.ProvideMany(context.Background(), nil); (err != nil) != tt.wantErr { - t.Errorf("ProvideManyWrapper.ProvideMany() error = %v, wantErr %v", err, tt.wantErr) - } - - if ready := pmw.Ready(); ready != tt.ready { - t.Errorf("ProvideManyWrapper.Ready() unexpected output = %v, want %v", ready, tt.ready) - } - }) - } -} - -func newDummyProvideMany(ready, failProviding bool) *dummyProvideMany { - return &dummyProvideMany{ - ready: ready, - failProviding: failProviding, - } -} - -type dummyProvideMany struct { - ready, failProviding bool -} - -func (dpm *dummyProvideMany) ProvideMany(ctx context.Context, keys []multihash.Multihash) error { - if dpm.failProviding { - return errors.New("error providing many") - } - - return nil -} -func (dpm *dummyProvideMany) Ready() bool { - return dpm.ready -} diff --git a/test/sharness/t0041-ping.sh b/test/sharness/t0041-ping.sh index c4665b9ba14..8fdfe17975e 100755 --- a/test/sharness/t0041-ping.sh +++ b/test/sharness/t0041-ping.sh @@ -27,7 +27,7 @@ test_expect_success "test ping other" ' test_expect_success "test ping unreachable peer" ' printf "Looking up peer %s\n" "$BAD_PEER" > bad_ping_exp && - printf "Error: peer lookup failed: routing: not found\n" >> bad_ping_exp && + printf "PING QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJx.\nPing error: routing: not found\nError: ping failed\n" >> bad_ping_exp && ! ipfsi 0 ping -n2 -- "$BAD_PEER" > bad_ping_actual 2>&1 && test_cmp bad_ping_exp bad_ping_actual ' diff --git a/test/sharness/t0701-delegated-routing-reframe.sh b/test/sharness/t0701-delegated-routing-reframe.sh index 424be6ef55e..5070b4fff16 100755 --- a/test/sharness/t0701-delegated-routing-reframe.sh +++ b/test/sharness/t0701-delegated-routing-reframe.sh @@ -66,13 +66,81 @@ test_expect_success "no routers means findprovs returns no results" ' test_kill_ipfs_daemon +ipfs config Routing.Type --json '"custom"' || exit 1 +ipfs config Routing.Methods --json '{ + "find-peers": { + "RouterName": "TestDelegatedRouter" + }, + "find-providers": { + "RouterName": "TestDelegatedRouter" + }, + "get-ipns": { + "RouterName": "TestDelegatedRouter" + }, + "provide": { + "RouterName": "TestDelegatedRouter" + } + }' || exit 1 + +test_expect_success "missing method params makes daemon fails" ' + echo "Error: constructing the node (see log for full detail): method name \"put-ipns\" is missing from Routing.Methods config param" > expected_error && + GOLOG_LOG_LEVEL=fatal ipfs daemon 2> actual_error || exit 0 && + test_cmp expected_error actual_error +' + +ipfs config Routing.Methods --json '{ + "find-peers": { + "RouterName": "TestDelegatedRouter" + }, + "find-providers": { + "RouterName": "TestDelegatedRouter" + }, + "get-ipns": { + "RouterName": "TestDelegatedRouter" + }, + "provide": { + "RouterName": "TestDelegatedRouter" + }, + "put-ipns": { + "RouterName": "TestDelegatedRouter" + }, + "NOT_SUPPORTED": { + "RouterName": "TestDelegatedRouter" + } + }' || exit 1 + +test_expect_success "having wrong methods makes daemon fails" ' + echo "Error: constructing the node (see log for full detail): method name \"NOT_SUPPORTED\" is not a supported method on Routing.Methods config param" > expected_error && + GOLOG_LOG_LEVEL=fatal ipfs daemon 2> actual_error || exit 0 && + test_cmp expected_error actual_error +' + # set Routing config to only use delegated routing via mocked reframe endpoint + +ipfs config Routing.Type --json '"custom"' || exit 1 ipfs config Routing.Routers.TestDelegatedRouter --json '{ "Type": "reframe", "Parameters": { "Endpoint": "http://127.0.0.1:5098/reframe" } }' || exit 1 +ipfs config Routing.Methods --json '{ + "find-peers": { + "RouterName": "TestDelegatedRouter" + }, + "find-providers": { + "RouterName": "TestDelegatedRouter" + }, + "get-ipns": { + "RouterName": "TestDelegatedRouter" + }, + "provide": { + "RouterName": "TestDelegatedRouter" + }, + "put-ipns": { + "RouterName": "TestDelegatedRouter" + } + }' || exit 1 test_expect_success "adding reframe endpoint to Routing.Routers config works" ' echo "http://127.0.0.1:5098/reframe" > expected &&