From 2a55fb5756ddc5a62684b70e4e369041b6fa85c8 Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 18 Aug 2023 21:36:23 -0700 Subject: [PATCH] Enable ipnisync to be served over libp2p. This should allow index providers such as boost to disable publishing over data-transfer/graphsync and be able to publish over libp2p without having to do any other configuration. --- engine/engine.go | 19 +++++++------------ go.mod | 6 ++++-- go.sum | 14 ++++++++++---- mirror/mirror.go | 12 +++++++----- mirror/mirror_env_test.go | 8 ++++++-- 5 files changed, 34 insertions(+), 25 deletions(-) diff --git a/engine/engine.go b/engine/engine.go index f1fa39ca..86637b81 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -148,23 +148,18 @@ func (e *Engine) newPublisher() (dagsync.Publisher, error) { log.Info("Remote announcements disabled; all advertisements will only be stored locally.") return nil, nil case HttpPublisher: - var httpPub *ipnisync.Publisher - var err error - if e.pubHttpWithoutServer { - httpPub, err = ipnisync.NewPublisher(e.pubHttpListenAddr, e.lsys, e.key, - ipnisync.WithHeadTopic(e.pubTopicName), - ipnisync.WithHandlerPath(e.pubHttpHandlerPath), - ipnisync.WithServer(false)) - } else { - httpPub, err = ipnisync.NewPublisher(e.pubHttpListenAddr, e.lsys, e.key, - ipnisync.WithHeadTopic(e.pubTopicName), - ipnisync.WithServer(true)) - } + httpPub, err := ipnisync.NewPublisher(e.lsys, e.key, + ipnisync.WithStreamHost(e.h), // TODO: Should serving HTTP over libp2p be configurable? + ipnisync.WithHTTPListenAddrs(e.pubHttpListenAddr), + ipnisync.WithHeadTopic(e.pubTopicName), + ipnisync.WithHandlerPath(e.pubHttpHandlerPath), + ipnisync.WithStartServer(!e.pubHttpWithoutServer)) if err != nil { return nil, fmt.Errorf("cannot create http publisher: %w", err) } return httpPub, nil case DataTransferPublisher: + log.Warn("Support ending for publishing IPNI data over data-transfer/graphsync, Disable this feature in configuration and test that indexing is working over libp2p.") if e.pubDT != nil { dtPub, err := dtsync.NewPublisherFromExisting(e.pubDT, e.h, e.pubTopicName, e.lsys, dtsync.WithAllowPeer(e.syncPolicy.Allowed)) if err != nil { diff --git a/go.mod b/go.mod index 410cf019..9b0d27fc 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 github.com/ipld/go-ipld-prime v0.20.0 - github.com/ipni/go-libipni v0.4.0 - github.com/libp2p/go-libp2p v0.29.2 + github.com/ipni/go-libipni v0.4.1-0.20230819032918-b91b8e8b2ff5 + github.com/libp2p/go-libp2p v0.29.1-0.20230819022641-d88fa5e2a696 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multiaddr v0.10.1 @@ -44,6 +44,8 @@ require ( github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-ipfs-files v0.3.0 // indirect diff --git a/go.sum b/go.sum index 469e5d55..db9da2d7 100644 --- a/go.sum +++ b/go.sum @@ -235,8 +235,14 @@ github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e/go.mod h1:I github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -344,8 +350,8 @@ github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0/go.mod h1:od github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= -github.com/ipni/go-libipni v0.4.0 h1:zZ8OU2N0D4iYt0E9jInbDapeh9bG10b5sBgqvScflNw= -github.com/ipni/go-libipni v0.4.0/go.mod h1:LxH6NUmEVruK3FjV2bFWfXKougX7AIe7wVjvPqITrDI= +github.com/ipni/go-libipni v0.4.1-0.20230819032918-b91b8e8b2ff5 h1:ALGas2QZaG/Imf72bq6C8joiuRB8Q3sonJAJS3uU1KI= +github.com/ipni/go-libipni v0.4.1-0.20230819032918-b91b8e8b2ff5/go.mod h1:EIfozn+eZNwtjf1ttMDhESOsoKe5ddkPOZROgORZrGU= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -393,8 +399,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.29.2 h1:uPw/c8hOxoLP/KhFnzlc5Ejqf+OmAL1dwIsqE31WBtY= -github.com/libp2p/go-libp2p v0.29.2/go.mod h1:OU7nSq0aEZMsV2wY8nXn1+XNNt9q2UiR8LjW3Kmp2UE= +github.com/libp2p/go-libp2p v0.29.1-0.20230819022641-d88fa5e2a696 h1:6RRECCAo+B06+b3mWRPunW9oEF5prrt+VGVAwvI0WC4= +github.com/libp2p/go-libp2p v0.29.1-0.20230819022641-d88fa5e2a696/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU= diff --git a/mirror/mirror.go b/mirror/mirror.go index 2939c32f..804b4fa3 100644 --- a/mirror/mirror.go +++ b/mirror/mirror.go @@ -75,11 +75,13 @@ func New(ctx context.Context, source peer.AddrInfo, o ...Option) (*Mirror, error } } - // Create ipnisync publisher. - // - // TODO: When libp2phttp available, Listen on http over libp2p if - // httpListenAddr is not set. - m.pub, err = ipnisync.NewPublisher(m.httpListenAddr, m.ls, m.privKey, ipnisync.WithHeadTopic(m.topic), ipnisync.WithServer(true)) + // Create ipnisync publisher. If m.httpListenAddr has a value, then mirror + // will serve over HTTP on that address. If there is a libp2p Host, then + // the mirror will serve HTTP over libp2p. + m.pub, err = ipnisync.NewPublisher(m.ls, m.privKey, + ipnisync.WithHTTPListenAddrs(m.httpListenAddr), + ipnisync.WithStreamHost(m.h), + ipnisync.WithHeadTopic(m.topic)) if err != nil { return nil, err } diff --git a/mirror/mirror_env_test.go b/mirror/mirror_env_test.go index ce5997da..697cdd05 100644 --- a/mirror/mirror_env_test.go +++ b/mirror/mirror_env_test.go @@ -60,10 +60,14 @@ func (te *testEnv) startMirror(t *testing.T, ctx context.Context, opts ...mirror te.mirrorSyncLs.SetReadStorage(te.mirrorSyncLsStore) te.mirrorSyncLs.SetWriteStorage(te.mirrorSyncLsStore) - te.mirrorSync = ipnisync.NewSync(te.mirrorSyncLs, nil, nil) + te.mirrorSync = ipnisync.NewSync(te.mirrorSyncLs, nil) require.NoError(t, err) t.Cleanup(func() { te.mirrorSync.Close() }) - te.mirrorSyncer, err = te.mirrorSync.NewSyncer(te.mirrorHost.ID(), te.mirror.PublisherAddrs()) + pubInfo := peer.AddrInfo{ + ID: te.mirrorHost.ID(), + Addrs: te.mirror.PublisherAddrs(), + } + te.mirrorSyncer, err = te.mirrorSync.NewSyncer(pubInfo) require.NoError(t, err) }