Enable ipnisync to be served over libp2p.

This should allow index providers such as boost to disable publishing over data-transfer/graphsync and be able to publish over libp2p without having to do any other configuration.

engine/engine.go

@@ -148,23 +148,18 @@ func (e *Engine) newPublisher() (dagsync.Publisher, error) {
 		log.Info("Remote announcements disabled; all advertisements will only be stored locally.")
 		return nil, nil
 	case HttpPublisher:
-		var httpPub *ipnisync.Publisher
-		var err error
-		if e.pubHttpWithoutServer {
-			httpPub, err = ipnisync.NewPublisher(e.pubHttpListenAddr, e.lsys, e.key,
-				ipnisync.WithHeadTopic(e.pubTopicName),
-				ipnisync.WithHandlerPath(e.pubHttpHandlerPath),
-				ipnisync.WithServer(false))
-		} else {
-			httpPub, err = ipnisync.NewPublisher(e.pubHttpListenAddr, e.lsys, e.key,
-				ipnisync.WithHeadTopic(e.pubTopicName),
-				ipnisync.WithServer(true))
-		}
+		httpPub, err := ipnisync.NewPublisher(e.lsys, e.key,
+			ipnisync.WithStreamHost(e.h), // TODO: Should serving HTTP over libp2p be configurable?
+			ipnisync.WithHTTPListenAddrs(e.pubHttpListenAddr),
+			ipnisync.WithHeadTopic(e.pubTopicName),
+			ipnisync.WithHandlerPath(e.pubHttpHandlerPath),
+			ipnisync.WithStartServer(!e.pubHttpWithoutServer))
 		if err != nil {
 			return nil, fmt.Errorf("cannot create http publisher: %w", err)
 		}
 		return httpPub, nil
 	case DataTransferPublisher:
+		log.Warn("Support ending for publishing IPNI data over data-transfer/graphsync, Disable this feature in configuration and test that indexing is working over libp2p.")
 		if e.pubDT != nil {
 			dtPub, err := dtsync.NewPublisherFromExisting(e.pubDT, e.h, e.pubTopicName, e.lsys, dtsync.WithAllowPeer(e.syncPolicy.Allowed))
 			if err != nil {

go.mod

@@ -18,8 +18,8 @@ require (
 	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0
 	github.com/ipld/go-ipld-prime v0.20.0
-	github.com/ipni/go-libipni v0.4.0
-	github.com/libp2p/go-libp2p v0.29.2
+	github.com/ipni/go-libipni v0.4.1-0.20230819032918-b91b8e8b2ff5
+	github.com/libp2p/go-libp2p v0.29.1-0.20230819022641-d88fa5e2a696
 	github.com/libp2p/go-libp2p-pubsub v0.9.3
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/multiformats/go-multiaddr v0.10.1
@@ -44,6 +44,8 @@ require (
 	github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect
 	github.com/ipfs/go-bitfield v1.1.0 // indirect
 	github.com/ipfs/go-ipfs-files v0.3.0 // indirect

go.sum

@@ -235,8 +235,14 @@ github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
+github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
@@ -344,8 +350,8 @@ github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0/go.mod h1:od
 github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
 github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
-github.com/ipni/go-libipni v0.4.0 h1:zZ8OU2N0D4iYt0E9jInbDapeh9bG10b5sBgqvScflNw=
-github.com/ipni/go-libipni v0.4.0/go.mod h1:LxH6NUmEVruK3FjV2bFWfXKougX7AIe7wVjvPqITrDI=
+github.com/ipni/go-libipni v0.4.1-0.20230819032918-b91b8e8b2ff5 h1:ALGas2QZaG/Imf72bq6C8joiuRB8Q3sonJAJS3uU1KI=
+github.com/ipni/go-libipni v0.4.1-0.20230819032918-b91b8e8b2ff5/go.mod h1:EIfozn+eZNwtjf1ttMDhESOsoKe5ddkPOZROgORZrGU=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
@@ -393,8 +399,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y
 github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic=
 github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
 github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
-github.com/libp2p/go-libp2p v0.29.2 h1:uPw/c8hOxoLP/KhFnzlc5Ejqf+OmAL1dwIsqE31WBtY=
-github.com/libp2p/go-libp2p v0.29.2/go.mod h1:OU7nSq0aEZMsV2wY8nXn1+XNNt9q2UiR8LjW3Kmp2UE=
+github.com/libp2p/go-libp2p v0.29.1-0.20230819022641-d88fa5e2a696 h1:6RRECCAo+B06+b3mWRPunW9oEF5prrt+VGVAwvI0WC4=
+github.com/libp2p/go-libp2p v0.29.1-0.20230819022641-d88fa5e2a696/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g=
 github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s=
 github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w=
 github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU=

mirror/mirror.go

@@ -75,11 +75,13 @@ func New(ctx context.Context, source peer.AddrInfo, o ...Option) (*Mirror, error
 		}
 	}
 
-	// Create ipnisync publisher.
-	//
-	// TODO: When libp2phttp available, Listen on http over libp2p if
-	// httpListenAddr is not set.
-	m.pub, err = ipnisync.NewPublisher(m.httpListenAddr, m.ls, m.privKey, ipnisync.WithHeadTopic(m.topic), ipnisync.WithServer(true))
+	// Create ipnisync publisher. If m.httpListenAddr has a value, then mirror
+	// will serve over HTTP on that address. If there is a libp2p Host, then
+	// the mirror will serve HTTP over libp2p.
+	m.pub, err = ipnisync.NewPublisher(m.ls, m.privKey,
+		ipnisync.WithHTTPListenAddrs(m.httpListenAddr),
+		ipnisync.WithStreamHost(m.h),
+		ipnisync.WithHeadTopic(m.topic))
 	if err != nil {
 		return nil, err
 	}

mirror/mirror_env_test.go

@@ -60,10 +60,14 @@ func (te *testEnv) startMirror(t *testing.T, ctx context.Context, opts ...mirror
 	te.mirrorSyncLs.SetReadStorage(te.mirrorSyncLsStore)
 	te.mirrorSyncLs.SetWriteStorage(te.mirrorSyncLsStore)
 
-	te.mirrorSync = ipnisync.NewSync(te.mirrorSyncLs, nil, nil)
+	te.mirrorSync = ipnisync.NewSync(te.mirrorSyncLs, nil)
 	require.NoError(t, err)
 	t.Cleanup(func() { te.mirrorSync.Close() })
-	te.mirrorSyncer, err = te.mirrorSync.NewSyncer(te.mirrorHost.ID(), te.mirror.PublisherAddrs())
+	pubInfo := peer.AddrInfo{
+		ID:    te.mirrorHost.ID(),
+		Addrs: te.mirror.PublisherAddrs(),
+	}
+	te.mirrorSyncer, err = te.mirrorSync.NewSyncer(pubInfo)
 	require.NoError(t, err)
 }