feat: Routing.Type=auto (DHT+IPNI)

cmd/ipfs/daemon.go

@@ -30,6 +30,7 @@ import (
 	fsrepo "github.com/ipfs/kubo/repo/fsrepo"
 	"github.com/ipfs/kubo/repo/fsrepo/migrations"
 	"github.com/ipfs/kubo/repo/fsrepo/migrations/ipfsfetcher"
+	pnet "github.com/libp2p/go-libp2p/core/pnet"
 	sockets "github.com/libp2p/go-socket-activation"
 
 	cmds "github.com/ipfs/go-ipfs-cmds"
@@ -61,6 +62,7 @@ const (
 	routingOptionNoneKwd      = "none"
 	routingOptionCustomKwd    = "custom"
 	routingOptionDefaultKwd   = "default"
+	routingOptionAutoKwd      = "auto"
 	unencryptTransportKwd     = "disable-transport-encryption"
 	unrestrictedAPIAccessKwd  = "unrestricted-api"
 	writableKwd               = "writable"
@@ -89,7 +91,7 @@ For example, to change the 'Gateway' port:
 
   ipfs config Addresses.Gateway /ip4/127.0.0.1/tcp/8082
 
-The API address can be changed the same way:
+The RPC API address can be changed the same way:
 
   ipfs config Addresses.API /ip4/127.0.0.1/tcp/5002
 
@@ -100,14 +102,14 @@ other computers in the network, use 0.0.0.0 as the ip address:
 
   ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/8080
 
-Be careful if you expose the API. It is a security risk, as anyone could
+Be careful if you expose the RPC API. It is a security risk, as anyone could
 control your node remotely. If you need to control the node remotely,
 make sure to protect the port as you would other services or database
 (firewall, authenticated proxy, etc).
 
 HTTP Headers
 
-ipfs supports passing arbitrary headers to the API and Gateway. You can
+ipfs supports passing arbitrary headers to the RPC API and Gateway. You can
 do this by setting headers on the API.HTTPHeaders and Gateway.HTTPHeaders
 keys:
 
@@ -141,18 +143,6 @@ environment variable:
 
   export IPFS_PATH=/path/to/ipfsrepo
 
-Routing
-
-IPFS by default will use a DHT for content routing. There is an alternative
-that operates the DHT in a 'client only' mode that can be enabled by
-running the daemon as:
-
-  ipfs daemon --routing=dhtclient
-
-Or you can set routing to dhtclient in the config:
-
-  ipfs config Routing.Type dhtclient
-
 DEPRECATION NOTICE
 
 Previously, ipfs used an environment variable as seen below:
@@ -402,14 +392,30 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 
 	routingOption, _ := req.Options[routingOptionKwd].(string)
 	if routingOption == routingOptionDefaultKwd {
-		routingOption = cfg.Routing.Type
+		routingOption = cfg.Routing.Type.WithDefault(routingOptionAutoKwd)
 		if routingOption == "" {
+			routingOption = routingOptionAutoKwd
+		}
+	}
+
+	// Private setups can't leverage peers returned by default IPNIs (Routing.Type=auto)
+	// To avoid breaking existing setups, switch them to DHT-only.
+	if routingOption == routingOptionAutoKwd {
+		if key, _ := repo.SwarmKey(); key != nil || pnet.ForcePrivateNetwork {
+			log.Error("Private networking (swarm.key / LIBP2P_FORCE_PNET) does not work with public HTTP IPNIs enabled by Routing.Type=auto. Kubo will use Routing.Type=dht instead. Update config to remove this message.")
 			routingOption = routingOptionDHTKwd
 		}
 	}
+
 	switch routingOption {
 	case routingOptionSupernodeKwd:
 		return errors.New("supernode routing was never fully implemented and has been removed")
+	case routingOptionDefaultKwd, routingOptionAutoKwd:
+		ncfg.Routing = libp2p.ConstructDefaultRouting(
+			cfg.Identity.PeerID,
+			cfg.Addresses.Swarm,
+			cfg.Identity.PrivKey,
+		)
 	case routingOptionDHTClientKwd:
 		ncfg.Routing = libp2p.DHTClientOption
 	case routingOptionDHTKwd:
@@ -446,6 +452,11 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 		fmt.Printf("Swarm key fingerprint: %x\n", node.PNetFingerprint)
 	}
 
+	if (pnet.ForcePrivateNetwork || node.PNetFingerprint != nil) && routingOption == routingOptionAutoKwd {
+		// This should never happen, but better safe than sorry
+		log.Fatal("Private network does not work with Routing.Type=auto. Update your config to Routing.Type=dht (or none, and do manual peering)")
+	}
+
 	printSwarmAddrs(node)
 
 	defer func() {

config/init.go

@@ -48,7 +48,7 @@ func InitWithIdentity(identity Identity) (*Config, error) {
 		},
 
 		Routing: Routing{
-			Type:    "dht",
+			Type:    nil,
 			Methods: nil,
 			Routers: nil,
 		},

config/profile.go

@@ -174,7 +174,7 @@ functionality - performance of content discovery and data
 fetching may be degraded.
 `,
 		Transform: func(c *Config) error {
-			c.Routing.Type = "dhtclient"
+			c.Routing.Type = NewOptionalString("dhtclient") // TODO: https://github.com/ipfs/kubo/issues/9480
 			c.AutoNAT.ServiceMode = AutoNATServiceDisabled
 			c.Reprovider.Interval = NewOptionalDuration(0)
 

config/reprovider.go

@@ -1,5 +1,10 @@
 package config
 
+import "time"
+
+const DefaultReproviderInterval = time.Hour * 22 // https://github.com/ipfs/kubo/pull/9326
+const DefaultReproviderStrategy = "all"
+
 type Reprovider struct {
 	Interval *OptionalDuration `json:",omitempty"` // Time period to reprovide locally stored objects to the network
 	Strategy *OptionalString   `json:",omitempty"` // Which keys to announce

config/routing.go

@@ -10,9 +10,10 @@ import (
 type Routing struct {
 	// Type sets default daemon routing mode.
 	//
-	// Can be one of "dht", "dhtclient", "dhtserver", "none", or "custom".
-	// When "custom" is set, you can specify a list of Routers.
-	Type string
+	// Can be one of "auto", "dht", "dhtclient", "dhtserver", "none", or "custom".
+	// When unset or set to "auto", DHT and implicit routers are used.
+	// When "custom" is set, user-provided Routing.Routers is used.
+	Type *OptionalString `json:",omitempty"`
 
 	Routers Routers
 

config/routing_test.go

@@ -13,7 +13,7 @@ func TestRouterParameters(t *testing.T) {
 	sec := time.Second
 	min := time.Minute
 	r := Routing{
-		Type: "custom",
+		Type: NewOptionalString("custom"),
 		Routers: map[string]RouterParser{
 			"router-dht": {Router{
 				Type: RouterTypeDHT,
@@ -113,7 +113,7 @@ func TestRouterMissingParameters(t *testing.T) {
 	require := require.New(t)
 
 	r := Routing{
-		Type: "custom",
+		Type: NewOptionalString("custom"),
 		Routers: map[string]RouterParser{
 			"router-wrong-reframe": {Router{
 				Type: RouterTypeReframe,

core/core_test.go

@@ -221,7 +221,7 @@ func GetNode(t *testing.T, reframeURLs ...string) *IpfsNode {
 			API:   []string{"/ip4/127.0.0.1/tcp/0"},
 		},
 		Routing: config.Routing{
-			Type:    "custom",
+			Type:    config.NewOptionalString("custom"),
 			Routers: routers,
 			Methods: config.Methods{
 				config.MethodNameFindPeers: config.Method{

core/node/groups.go

@@ -294,8 +294,8 @@ func Online(bcfg *BuildCfg, cfg *config.Config) fx.Option {
 		OnlineProviders(
 			cfg.Experimental.StrategicProviding,
 			cfg.Experimental.AcceleratedDHTClient,
-			cfg.Reprovider.Strategy.WithDefault(DefaultReproviderStrategy),
-			cfg.Reprovider.Interval.WithDefault(DefaultReproviderInterval),
+			cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy),
+			cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval),
 		),
 	)
 }
@@ -312,8 +312,8 @@ func Offline(cfg *config.Config) fx.Option {
 		OfflineProviders(
 			cfg.Experimental.StrategicProviding,
 			cfg.Experimental.AcceleratedDHTClient,
-			cfg.Reprovider.Strategy.WithDefault(DefaultReproviderStrategy),
-			cfg.Reprovider.Interval.WithDefault(DefaultReproviderInterval),
+			cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy),
+			cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval),
 		),
 	)
 }

core/node/libp2p/routingopt.go

@@ -2,6 +2,7 @@ package libp2p
 
 import (
 	"context"
+	"time"
 
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/kubo/config"
@@ -23,6 +24,63 @@ type RoutingOption func(
 	...peer.AddrInfo,
 ) (routing.Routing, error)
 
+// Default HTTP routers used in parallel to DHT when Routing.Type = "auto"
+var defaultHTTPRouters = []string{
+	"https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
+	// TODO: add an independent router from Cloudflare
+}
+
+// ConstructDefaultRouting returns routers used when Routing.Type is unset or set to "auto"
+func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func(
+	ctx context.Context,
+	host host.Host,
+	dstore datastore.Batching,
+	validator record.Validator,
+	bootstrapPeers ...peer.AddrInfo,
+) (routing.Routing, error) {
+	return func(
+		ctx context.Context,
+		host host.Host,
+		dstore datastore.Batching,
+		validator record.Validator,
+		bootstrapPeers ...peer.AddrInfo,
+	) (routing.Routing, error) {
+		// Defined routers will be queried in parallel (optimizing for response speed)
+		// Different trade-offs can be made by setting Routing.Type = "custom" with own Routing.Routers
+		var routers []*routinghelpers.ParallelRouter
+
+		// Run the default DHT routing (same as Routing.Type = "dht")
+		dhtRouting, err := DHTOption(ctx, host, dstore, validator, bootstrapPeers...)
+		if err != nil {
+			return nil, err
+		}
+		routers = append(routers, &routinghelpers.ParallelRouter{
+			Router:       dhtRouting,
+			IgnoreError:  false,
+			Timeout:      5 * time.Minute, // https://github.com/ipfs/kubo/pull/9475#discussion_r1042501333
+			ExecuteAfter: 0,
+		})
+
+		// Append HTTP routers for additional speed
+		for _, endpoint := range defaultHTTPRouters {
+			httpRouter, err := irouting.ConstructHTTPRouter(endpoint, peerID, addrs, privKey)
+			if err != nil {
+				return nil, err
+			}
+			routers = append(routers, &routinghelpers.ParallelRouter{
+				Router:       httpRouter,
+				IgnoreError:  true,             // https://github.com/ipfs/kubo/pull/9475#discussion_r1042507387
+				Timeout:      15 * time.Second, // 5x server value from https://github.com/ipfs/kubo/pull/9475#discussion_r1042428529
+				ExecuteAfter: 0,
+			})
+		}
+
+		routing := routinghelpers.NewComposableParallel(routers)
+		return routing, nil
+	}
+}
+
+// constructDHTRouting is used when Routing.Type = "dht"
 func constructDHTRouting(mode dht.ModeOpt) func(
 	ctx context.Context,
 	host host.Host,
@@ -49,6 +107,7 @@ func constructDHTRouting(mode dht.ModeOpt) func(
 	}
 }
 
+// ConstructDelegatedRouting is used when Routing.Type = "custom"
 func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs []string, privKey string) func(
 	ctx context.Context,
 	host host.Host,

core/node/provider.go

@@ -18,9 +18,6 @@ import (
 	irouting "github.com/ipfs/kubo/routing"
 )
 
-const DefaultReproviderInterval = time.Hour * 22 // https://github.com/ipfs/kubo/pull/9326
-const DefaultReproviderStrategy = "all"
-
 // SIMPLE
 
 // ProviderQueue creates new datastore backed provider queue

docs/changelogs/v0.18.md

@@ -11,6 +11,7 @@ Below is an outline of all that is in this release, so you get a sense of all th
     - [Overview](#overview)
     - [🔦 Highlights](#-highlights)
       - [(DAG-)JSON and (DAG-)CBOR Response Formats on Gateways](#dag-json-and-dag-cbor-response-formats-on-gateways)
+      - [Content Routing](#content-routing)
       - [Increased `Reprovider.Interval`](#increased-reproviderinterval)
     - [Changelog](#changelog)
     - [Contributors](#contributors)
@@ -22,7 +23,7 @@ Below is an outline of all that is in this release, so you get a sense of all th
 Implemented [IPIP-328](https://github.com/ipfs/specs/pull/328) which adds support
 to DAG-JSON and DAG-CBOR, as well as their non-DAG variants, to the gateway. Now,
 CIDs that encode JSON, CBOR, DAG-JSON and DAG-CBOR objects can be retrieved, and
-traversed through IPLD Links.
+traversed thanks to the [special meaning of CBOR Tag 42](https://github.com/ipld/cid-cbor/).
 
 HTTP clients can request JSON, CBOR, DAG-JSON, and DAG-CBOR responses by either
 passing the query parameter `?format` or setting the `Accept` HTTP header to the
@@ -69,6 +70,24 @@ $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq
 }
 ```
 
+
+#### Content Routing
+
+Content routing is the process of discovering which peers provide a piece of content. Kubo has traditionally only supported [libp2p's implementation of Kademlia DHT](https://github.com/libp2p/specs/tree/master/kad-dht) for content routing.
+
+Kubo can now bridge networks by including support for the [delegated routing HTTP API](https://github.com/ipfs/specs/pull/337). Users can compose content routers using the `Routing.Routers` config, to pick content routers with different tradeoffs than a Kademlia DHT (for example, high-performance and high-capacity centralized endpoints, dedicated Kademlia DHT nodes, routers with unique provider records, privacy-focused content routers, etc.).
+
+One example is [InterPlanetary Network Indexers](https://github.com/ipni/specs/blob/main/IPNI.md#readme), which are HTTP endpoints that cache records from both the IPFS network and other sources such as web3.storage and Filecoin. This improves not only content availability by enabling Kubo to transparently fetch content directly from Filecoin storage providers, but also improves IPFS content routing latency by an order of magnitude and decreases resource consumption.
+*Note:* it's possible to retrieve content stored by Filecoin Storage Providers (SPs) from Kubo if the SPs service Bitswap requests.  As of this release, some SPs are advertising Bitswap.  You can follow the roadmap progress for IPNIs and Bitswap in SPs [here](https://www.starmaps.app/roadmap/github.com/protocol/bedrock/issues/1).
+
+In this release, the default content router is changed from `dht` to `auto`. The `auto` router includes the IPFS DHT in addition to the [cid.contact](https://cid.contact) IPNI instance. In future releases, we plan to expand the functionality of `auto` to encompass automatic discovery of content routers, which will improve performance and content availability (for example, see [IPIP-342](https://github.com/ipfs/specs/pull/342)).
+
+Previous behavior can be restored by setting `Routing.Type` to `dht`.
+
+Alternative routing rules, including alternative IPNI endpoints, can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`.
+
+Learn more in [`Routing` docs](https://github.com/ipfs/kubo/blob/master/docs/config.md#routing).
+
 #### Increased `Reprovider.Interval`
 
 Default changed from 12h to 22h.
@@ -79,6 +98,11 @@ and [kubo#9326](https://github.com/ipfs/kubo/pull/9326).
 
 Learn more: [`Reprovider` config](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#reprovider)
 
+#### Lowered `ConnMgr`
+
+<!-- TODO: https://github.com/ipfs/kubo/pull/9483 -->
+
+
 ### Changelog
 
 ### Contributors