Update test to use libp2phttp publisher

- Update comments
- Use latest go-libp2p

cmd/provider/internal/config/httppublisher.go

@@ -11,9 +11,11 @@ type HttpPublisher struct {
 	// specified, the ListenMultiaddr is used.
 	AnnounceMultiaddr string
 	// ListenMultiaddr is the address of the interface to listen for HTTP
-	// requests for advertisements.
+	// requests for advertisements. Set this to "" to disable serving plain
+	// HTTP if only libp2phttp is wanted.
 	ListenMultiaddr string
-	// NoLibp2p disables serving HTTP over libp2p if true.
+	// NoLibp2p disables serving HTTP over libp2p if true. Set this to true to
+	// publish over plain HTTP only.
 	NoLibp2p bool
 }
 

cmd/provider/internal/config/ingest.go

@@ -35,9 +35,9 @@ type Ingest struct {
 	HttpPublisher HttpPublisher
 
 	// PublisherKind specifies which dagsync.Publisher implementation to use.
-	// When set to "http", publisher serves HTTP over libp2p if
-	// HttpPublisher.NoLibp2p is false. If HttpPublisher.ListenMultiaddr
-	// publisher serves plain HTTP on that address.
+	// When set to "http", the publisher serves plain HTTP and libp2phttp.
+	// Libp2phttp is disabled by setting HttpPublisher.NoLibp2p to true, and
+	// plain HTTP is disabled by setting HttpPublisher.ListenMultiaddr to "".
 	PublisherKind PublisherKind
 
 	// SyncPolicy configures which indexers are allowed to sync advertisements

engine/engine_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
-	dssync "github.com/ipfs/go-datastore/sync"
 	leveldb "github.com/ipfs/go-ds-leveldb"
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
@@ -21,8 +20,7 @@ import (
 	"github.com/ipld/go-ipld-prime/traversal/selector"
 	selectorbuilder "github.com/ipld/go-ipld-prime/traversal/selector/builder"
 	"github.com/ipni/go-libipni/announce/message"
-	"github.com/ipni/go-libipni/dagsync/dtsync"
-	"github.com/ipni/go-libipni/dagsync/p2p/protocol/head"
+	"github.com/ipni/go-libipni/dagsync/ipnisync"
 	"github.com/ipni/go-libipni/ingest/schema"
 	"github.com/ipni/go-libipni/metadata"
 	"github.com/ipni/go-libipni/test"
@@ -108,7 +106,7 @@ func TestEngine_PublishLocal(t *testing.T) {
 	require.Equal(t, gotLatestAdCid, gotPublishedAdCid)
 }
 
-func TestEngine_PublishWithDataTransferPublisher(t *testing.T) {
+func TestEngine_PublishWithLibp2pHttpPublisher(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
 	t.Cleanup(cancel)
 
@@ -118,11 +116,17 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) {
 	// Use test name as gossip topic name for uniqueness per test.
 	topic := t.Name()
 
-	subHost, err := libp2p.New()
+	pubHost, err := libp2p.New()
 	require.NoError(t, err)
+	t.Cleanup(func() {
+		pubHost.Close()
+	})
 
-	pubHost, err := libp2p.New()
+	subHost, err := libp2p.New()
 	require.NoError(t, err)
+	t.Cleanup(func() {
+		subHost.Close()
+	})
 
 	// DirectConnectTicks set to 5 here, because if the gossub for the subHost
 	// does not start within n ticks then they never peer. So, if
@@ -163,7 +167,7 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) {
 			http.Error(w, err.Error(), 400)
 			return
 		}
-		// Since the message is coming from a dtsync publisher, the addresses
+		// Since the message is coming from a libp2p publisher, the addresses
 		// should include a p2p ID.
 		ais, err := peer.AddrInfosFromP2pAddrs(addrs...)
 		if err != nil {
@@ -186,23 +190,28 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) {
 		}
 		w.WriteHeader(http.StatusNoContent)
 	}))
-	defer ts.Close()
+	t.Cleanup(func() {
+		ts.Close()
+	})
 
 	pubT, err := pubG.Join(topic)
 	require.NoError(t, err)
 
 	subject, err := engine.New(
 		engine.WithDirectAnnounce(ts.URL),
 		engine.WithHost(pubHost),
-		engine.WithPublisherKind(engine.DataTransferPublisher),
+		engine.WithPublisherKind(engine.HttpPublisher),
+		engine.WithHttpPublisherListenAddr(""), // disable plain http
 		engine.WithTopic(pubT),
 		engine.WithTopicName(topic),
 		engine.WithExtraGossipData(wantExtraGossipData),
 	)
 	require.NoError(t, err)
 	err = subject.Start(ctx)
 	require.NoError(t, err)
-	defer subject.Shutdown()
+	t.Cleanup(func() {
+		subject.Shutdown()
+	})
 
 	subG, err := pubsub.NewGossipSub(ctx, subHost,
 		pubsub.WithDirectConnectTicks(1),
@@ -282,19 +291,20 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) {
 	require.NoError(t, err)
 	requireEqualDagsyncMessage(t, wantMessage, gotMessage)
 
-	gotRootCid, err := head.QueryRootCid(ctx, subHost, topic, pubHost.ID())
-	require.NoError(t, err)
-	require.Equal(t, gotPublishedAdCid, gotRootCid)
-
-	ds := dssync.MutexWrap(datastore.NewMapDatastore())
 	ls := cidlink.DefaultLinkSystem()
 	store := &memstore.Store{}
 	ls.SetReadStorage(store)
 	ls.SetWriteStorage(store)
 
-	sync, err := dtsync.NewSync(subHost, ds, ls, nil, 0, 0)
+	sync := ipnisync.NewSync(ls, nil, ipnisync.ClientStreamHost(subHost))
+	t.Cleanup(func() {
+		sync.Close()
+	})
+	subjectInfo := peer.AddrInfo{
+		ID: subject.Host().ID(),
+	}
+	syncer, err := sync.NewSyncer(subjectInfo)
 	require.NoError(t, err)
-	syncer := sync.NewSyncer(subject.Host().ID(), topic)
 	gotHead, err := syncer.GetHead(ctx)
 	require.NoError(t, err)
 	require.Equal(t, gotLatestAdCid, gotHead)
@@ -453,10 +463,10 @@ func TestEngine_ProducesSingleChainForMultipleProviders(t *testing.T) {
 	wantContextID1 := []byte("fish")
 	wantContextID2 := []byte("bird")
 	subject.RegisterMultihashLister(func(ctx context.Context, p peer.ID, contextID []byte) (provider.MultihashIterator, error) {
-
 		if string(contextID) == string(wantContextID1) && p == provider1id {
 			return provider.SliceMultihashIterator(mhs1), nil
-		} else if string(contextID) == string(wantContextID2) && p == provider2id {
+		}
+		if string(contextID) == string(wantContextID2) && p == provider2id {
 			return provider.SliceMultihashIterator(mhs2), nil
 		}
 		return nil, errors.New("not found")

engine/options.go

@@ -227,8 +227,11 @@ func WithEntriesCacheCapacity(s int) Option {
 	}
 }
 
-// WithPublisherKind sets the kind of publisher used to announce new advertisements.
-// If unset, advertisements are only stored locally and no announcements are made.
+// WithPublisherKind sets the kind of publisher used to serve advertisements.
+// If unset, advertisements are only stored locally and no announcements are
+// made. This does not affect the methods used to send announcements of new
+// advertisements, which are configured independent of this.
+//
 // See: PublisherKind.
 func WithPublisherKind(k PublisherKind) Option {
 	return func(o *options) error {
@@ -237,10 +240,12 @@ func WithPublisherKind(k PublisherKind) Option {
 	}
 }
 
-// WithHttpPublisherListenAddr sets the net listen address for the HTTP publisher.
-// If unset, the default net listen address of '0.0.0.0:3104' is used.
+// WithHttpPublisherListenAddr sets the net listen address for the HTTP
+// publisher. If unset, the default net listen address of '0.0.0.0:3104' is
+// used. To disable plain HTTP and only serve libp2phttp, explicitly set this
+// to "".
 //
-// Note that this option only takes effect if the PublisherKind is set to HttpPublisher.
+// This option only takes effect if the PublisherKind is set to HttpPublisher.
 // See: WithPublisherKind.
 func WithHttpPublisherListenAddr(addr string) Option {
 	return func(o *options) error {
@@ -250,7 +255,7 @@ func WithHttpPublisherListenAddr(addr string) Option {
 }
 
 // WithHttpNoLibp2p disables serving HTTP over libp2p if true and using an HTTP
-// publisher.
+// publisher. This is used to disable libp2phttp and only serve plain HTTP.
 func WithHttpNoLibp2p(disable bool) Option {
 	return func(o *options) error {
 		o.pubHttpNoLibp2p = disable

go.mod

@@ -18,8 +18,8 @@ require (
 	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0
 	github.com/ipld/go-ipld-prime v0.20.0
-	github.com/ipni/go-libipni v0.4.1-0.20230824030625-0669ee8d4bbe
-	github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19
+	github.com/ipni/go-libipni v0.4.1-0.20230826004428-2194d3a10198
+	github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee
 	github.com/libp2p/go-libp2p-pubsub v0.9.3
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/multiformats/go-multiaddr v0.10.1

go.sum

@@ -350,8 +350,8 @@ github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0/go.mod h1:od
 github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
 github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
-github.com/ipni/go-libipni v0.4.1-0.20230824030625-0669ee8d4bbe h1:wEg//rXzpRqDE9xETCkxMCobUq6c911aS4o8veqUxys=
-github.com/ipni/go-libipni v0.4.1-0.20230824030625-0669ee8d4bbe/go.mod h1:fVvfLxFC9MTiCUfaxYgvc9eoPGNYpzzVAL0taKy2n70=
+github.com/ipni/go-libipni v0.4.1-0.20230826004428-2194d3a10198 h1:uc4Pey8/KQa6hidvotIwvQrr59u5kU7AzN/8bKlVkL0=
+github.com/ipni/go-libipni v0.4.1-0.20230826004428-2194d3a10198/go.mod h1:fnTwBp3bJqWV6ItVmT44j9JpIKBuphBpDmACmHxavXw=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
@@ -399,8 +399,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y
 github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic=
 github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
 github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
-github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19 h1:oBCMCeLFvvHhAsLy9/9ybP71Sc5/DSGtZKn9aSLrwwk=
-github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g=
+github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee h1:UioCkyX/qjEs5mzVoBfZUChXtyhtXG9salRwQ8iUDfI=
+github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g=
 github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s=
 github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w=
 github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU=