Add distance tracking functionality to provider command

README.md

@@ -75,11 +75,11 @@ ipni ads dist  --ai=/ip4/76.219.232.45/tcp/24001/p2p/12D3KooWPNbkEgjdBNeaCGpsgCr
 ### `find`
 - Ask cid.contact where to find CID `bafybeigvgzoolc3drupxhlevdp2ugqcrbcsqfmcek2zxiw5wctk3xjpjwy`:
 ```sh
-ipni find -i cid.contact --cid bafybeigvgzoolc3drupxhlevdp2ugqcrbcsqfmcek2zxiw5wctk3xjpjwy
+ipni find -i https://cid.contact --cid bafybeigvgzoolc3drupxhlevdp2ugqcrbcsqfmcek2zxiw5wctk3xjpjwy
 ```
 - Ask cid.contact where to find multiple multihashes:
 ```sh
-./ipni find -i cid.contact \
+./ipni find -i https://cid.contact \
     --mh=2Drjgb5kxWdcTNfhfEC8F3Ltk4s16aAgG2aLnXxSdpiGTazLGE \
     --mh=2Drjgb4GmZ3cJGRunHYdHrmtgbmGoDuSMeN42gdU1jSiGmHVmA \
     --mh=2DrjgbJZxQgMTvWDG6ih2SNESWeoabccawmLwuFt1T59joGFxd
@@ -88,30 +88,30 @@ ipni find -i cid.contact --cid bafybeigvgzoolc3drupxhlevdp2ugqcrbcsqfmcek2zxiw5w
 ### `provider`
 - Get all providers known by the indexer dev.cid.contact:
 ```
-ipni provider -i dev.cid.contact --all
+ipni provider -i https://dev.cid.contact --all
 ```
 - Get information about the provider with ID `QmQzqxhK82kAmKvARFZSkUVS6fo9sySaiogAnx5EnZ6ZmC`
 ```
-ipni provider -i cid.contact -pid QmQzqxhK82kAmKvARFZSkUVS6fo9sySaiogAnx5EnZ6ZmC
+ipni provider -i https://cid.contact -pid QmQzqxhK82kAmKvARFZSkUVS6fo9sySaiogAnx5EnZ6ZmC
 ```
 ```
-echo QmQzqxhK82kAmKvARFZSkUVS6fo9sySaiogAnx5EnZ6ZmC | ipni provider -i cid.contact
+echo QmQzqxhK82kAmKvARFZSkUVS6fo9sySaiogAnx5EnZ6ZmC | ipni provider -i https://cid.contact
 ```
 - Get information about the providers returned from find results:
 ```sh
-ipni find -i cid.contact --cid bafybeigvgzoolc3drupxhlevdp2ugqcrbcsqfmcek2zxiw5wctk3xjpjwy --id-only | ipni provider -i cid.contact
+ipni find -i https://cid.contact --cid bafybeigvgzoolc3drupxhlevdp2ugqcrbcsqfmcek2zxiw5wctk3xjpjwy --id-only | ipni provider -i https://cid.contact
 ```
 - See which providers cid.contact knows about that dev.cid.contact does not:
 ```sh 
-ipni provider --all -i dev.cid.contact -id | ipni provider -invert -i cid.contact -id
+ipni provider --all -i https://dev.cid.contact -id | ipni provider -invert -i https://cid.contact -id
 ```
 - Get combined information from multiple providers:
 ```
-$ ipni provider --all -i alva.dev.cid.contact -i cora.dev.cid.contact --id-only | wc -l
+$ ipni provider --all -i https://alva.dev.cid.contact -i https://cora.dev.cid.contact --id-only | wc -l
      405
-$ ipni provider --all -i alva.dev.cid.contact --id-only | wc -l
+$ ipni provider --all -i https://alva.dev.cid.contact --id-only | wc -l
      209
-> ipni provider --all -i cora.dev.cid.contact --id-only | wc -l
+> ipni provider --all -i https://cora.dev.cid.contact --id-only | wc -l
      196
 ```
 
@@ -124,7 +124,7 @@ $ ipni provider --all -i alva.dev.cid.contact --id-only | wc -l
 ### `verify ingest`
 - Verfy ingestion at cid.contact, of multihashes 
 ```
-./ipni verify ingest -i cid.contact \
+./ipni verify ingest -i https://cid.contact \
     --ad-cid=baguqeerank3iclae2u4lin3vj2avuory3ny67tldh2cd5uodsgsdl6uawz3a \
     --provider-id=12D3KooWPNbkEgjdBNeaCGpsgCrPRETe4uBZf1ShFXStobdN18ys \
     --batch-size=25 \

cmd/ipni/ipni.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/ipni/ipni-cli"
 	"github.com/ipni/ipni-cli/pkg/ads"
 	"github.com/ipni/ipni-cli/pkg/find"
@@ -14,6 +15,9 @@ import (
 )
 
 func main() {
+	// Disable logging that happens in packages such as data-transfer.
+	_ = logging.SetLogLevel("*", "fatal")
+
 	app := &cli.App{
 		Name:    "ipni",
 		Usage:   "Commands to interact with IPNI indexers and index providers",

go.mod

@@ -6,9 +6,10 @@ require (
 	github.com/filecoin-project/go-address v1.1.0
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-datastore v0.6.0
+	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipld/go-car/v2 v2.10.0
 	github.com/ipld/go-ipld-prime v0.20.0
-	github.com/ipni/go-libipni v0.2.10
+	github.com/ipni/go-libipni v0.2.12
 	github.com/libp2p/go-libp2p v0.28.1
 	github.com/mattn/go-isatty v0.0.19
 	github.com/montanaflynn/stats v0.7.0
@@ -74,7 +75,6 @@ require (
 	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
 	github.com/ipfs/go-libipfs v0.6.2 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
-	github.com/ipfs/go-log/v2 v2.5.1 // indirect
 	github.com/ipfs/go-merkledag v0.10.0 // indirect
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect

go.sum

@@ -286,8 +286,8 @@ github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvB
 github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
 github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
-github.com/ipni/go-libipni v0.2.10 h1:XDEchmQwkrD/c67Fk3+dVcpdxcKPz5NbsPSAAjqF2Bs=
-github.com/ipni/go-libipni v0.2.10/go.mod h1:dhBH9HwxT6HzQPRZ8ikWv+ccqF8ucMIoGiiTSrHA4tw=
+github.com/ipni/go-libipni v0.2.12 h1:vR8fuUUdqTpBrykojxV7uFdkHLOV8RmtUtSFTQJR0yk=
+github.com/ipni/go-libipni v0.2.12/go.mod h1:dhBH9HwxT6HzQPRZ8ikWv+ccqF8ucMIoGiiTSrHA4tw=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
 github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=

pkg/adpub/client.go

@@ -25,13 +25,15 @@ import (
 type Client interface {
 	GetAdvertisement(context.Context, cid.Cid) (*Advertisement, error)
 	Close() error
-	Distance(context.Context, cid.Cid, cid.Cid) (int, error)
+	ClearStore()
+	Distance(context.Context, cid.Cid, cid.Cid) (int, cid.Cid, error)
 	List(context.Context, cid.Cid, int, io.Writer) error
 	SyncEntriesWithRetry(context.Context, cid.Cid) error
 }
 
 type client struct {
-	entriesDepthLimit selector.RecursionLimit
+	adChainDepthLimit int64
+	entriesDepthLimit int64
 	maxSyncRetry      uint64
 	syncRetryBackoff  time.Duration
 
@@ -72,6 +74,7 @@ func NewClient(addrInfo peer.AddrInfo, options ...Option) (Client, error) {
 		})).Node()
 
 	return &client{
+		adChainDepthLimit: opts.adChainDepthLimit,
 		entriesDepthLimit: opts.entriesDepthLimit,
 		maxSyncRetry:      opts.maxSyncRetry,
 		syncRetryBackoff:  opts.syncRetryBackoff,
@@ -91,26 +94,29 @@ func selectEntriesWithLimit(limit selector.RecursionLimit) datamodel.Node {
 		})).Node()
 }
 
-func (c *client) Distance(ctx context.Context, oldestCid, newestCid cid.Cid) (int, error) {
+func (c *client) Distance(ctx context.Context, oldestCid, newestCid cid.Cid) (int, cid.Cid, error) {
 	if oldestCid == cid.Undef {
-		return 0, errors.New("must specify a oldest CID")
+		return 0, cid.Undef, errors.New("must specify a oldest CID")
 	}
 
 	// Sync the advertisement without entries first.
-	var err error
-	_, err = c.syncAdWithRetry(ctx, oldestCid)
+	_, err := c.syncAdWithRetry(ctx, oldestCid)
 	if err != nil {
-		return 0, err
+		return 0, cid.Undef, err
 	}
 
 	// Load the synced advertisement from local store.
 	ad, err := c.store.getAdvertisement(ctx, oldestCid)
 	if err != nil {
-		return 0, err
+		return 0, cid.Undef, err
 	}
 
-	// TODO: Allow a maximum depth to be specified for the ad chain.
-	rLimit := selector.RecursionLimitNone()
+	var rLimit selector.RecursionLimit
+	if c.adChainDepthLimit == 0 {
+		rLimit = selector.RecursionLimitNone()
+	} else {
+		rLimit = selector.RecursionLimitDepth(c.adChainDepthLimit)
+	}
 
 	stopAt := cidlink.Link{Cid: ad.PreviousID}
 
@@ -124,10 +130,15 @@ func (c *client) Distance(ctx context.Context, oldestCid, newestCid cid.Cid) (in
 
 	newestCid, err = c.sub.Sync(ctx, c.publisher, newestCid, sel)
 	if err != nil {
-		return 0, err
+		return 0, cid.Undef, err
+	}
+
+	dist, err := c.store.distance(ctx, oldestCid, newestCid, c.adChainDepthLimit)
+	if err != nil {
+		return 0, cid.Undef, err
 	}
 
-	return c.store.distance(ctx, oldestCid, newestCid)
+	return dist, newestCid, nil
 }
 
 func (c *client) List(ctx context.Context, latestCid cid.Cid, n int, w io.Writer) error {
@@ -175,6 +186,9 @@ func (c *client) GetAdvertisement(ctx context.Context, adCid cid.Cid) (*Advertis
 }
 
 func (c *client) syncAdWithRetry(ctx context.Context, adCid cid.Cid) (cid.Cid, error) {
+	if c.maxSyncRetry == 0 {
+		return c.sub.Sync(ctx, c.publisher, adCid, c.adSel)
+	}
 	var attempt uint64
 	var err error
 	for {
@@ -199,7 +213,13 @@ func (c *client) syncAdWithRetry(ctx context.Context, adCid cid.Cid) (cid.Cid, e
 
 func (c *client) SyncEntriesWithRetry(ctx context.Context, id cid.Cid) error {
 	var attempt uint64
-	recurLimit := c.entriesDepthLimit
+	var recurLimit selector.RecursionLimit
+	if c.entriesDepthLimit == 0 {
+		recurLimit = selector.RecursionLimitNone()
+	} else {
+		recurLimit = selector.RecursionLimitDepth(c.entriesDepthLimit)
+	}
+
 	for {
 		sel := selectEntriesWithLimit(recurLimit)
 		_, err := c.sub.Sync(ctx, c.publisher, id, sel)
@@ -245,3 +265,7 @@ func (c *client) findNextMissingChunkLink(ctx context.Context, next cid.Cid) (ci
 func (c *client) Close() error {
 	return c.sub.Close()
 }
+
+func (c *client) ClearStore() {
+	c.store.clear()
+}

pkg/adpub/client_store.go

@@ -3,6 +3,7 @@ package adpub
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 
 	"github.com/ipfs/go-cid"
@@ -168,7 +169,7 @@ func (s *ClientStore) getAdvertisement(ctx context.Context, id cid.Cid) (*Advert
 	return a, nil
 }
 
-func (s *ClientStore) distance(ctx context.Context, oldestCid, newestCid cid.Cid) (int, error) {
+func (s *ClientStore) distance(ctx context.Context, oldestCid, newestCid cid.Cid, depthLimit int64) (int, error) {
 	var count int
 	for newestCid != oldestCid {
 		val, err := s.Batching.Get(ctx, datastore.NewKey(newestCid.String()))
@@ -199,6 +200,10 @@ func (s *ClientStore) distance(ctx context.Context, oldestCid, newestCid cid.Cid
 			break
 		}
 		newestCid = ad.PreviousID.(cidlink.Link).Cid
+
+		if count == int(depthLimit) {
+			return 0, fmt.Errorf("exceeded limit %d+", depthLimit)
+		}
 	}
 	return count, nil
 }
@@ -241,3 +246,7 @@ func (s *ClientStore) list(ctx context.Context, nextCid cid.Cid, n int, w io.Wri
 	}
 	return nil
 }
+
+func (s *ClientStore) clear() {
+	s.Batching = dssync.MutexWrap(datastore.NewMapDatastore())
+}

pkg/adpub/options.go

@@ -4,12 +4,16 @@ import (
 	"errors"
 	"fmt"
 	"time"
+)
 
-	"github.com/ipld/go-ipld-prime/traversal/selector"
+const (
+	defaultAdChainDepthLimit = 10000
+	defaultEntriesDepthLimit = 1000
 )
 
 type config struct {
-	entriesDepthLimit selector.RecursionLimit
+	adChainDepthLimit int64
+	entriesDepthLimit int64
 	maxSyncRetry      uint64
 	syncRetryBackoff  time.Duration
 	topic             string
@@ -21,9 +25,9 @@ type Option func(*config) error
 // getOpts creates a config and applies Options to it.
 func getOpts(opts []Option) (config, error) {
 	cfg := config{
-		entriesDepthLimit: selector.RecursionLimitNone(),
+		adChainDepthLimit: defaultAdChainDepthLimit,
+		entriesDepthLimit: defaultEntriesDepthLimit,
 		topic:             "/indexer/ingest/mainnet",
-		maxSyncRetry:      3,
 		syncRetryBackoff:  500 * time.Millisecond,
 	}
 
@@ -35,6 +39,18 @@ func getOpts(opts []Option) (config, error) {
 	return cfg, nil
 }
 
+// WithAdChainDepthLimit sets the depth limit when syncing an advertisement
+// chain. Setting to 0 means no limit.
+func WithAdChainDepthLimit(limit int64) Option {
+	return func(c *config) error {
+		if limit < 0 {
+			return errors.New("ad chain depth limit cannot be negative")
+		}
+		c.adChainDepthLimit = limit
+		return nil
+	}
+}
+
 // WithSyncRetryBackoff sets the length of time to wait before retrying a faild
 // sync. Defaults to 500ms if unset.
 func WithSyncRetryBackoff(d time.Duration) Option {
@@ -69,11 +85,7 @@ func WithEntriesDepthLimit(depthLimit int64) Option {
 		if depthLimit < 0 {
 			return errors.New("ad entries depth limit cannot be negative")
 		}
-		if depthLimit == 0 {
-			c.entriesDepthLimit = selector.RecursionLimitNone()
-		} else {
-			c.entriesDepthLimit = selector.RecursionLimitDepth(depthLimit)
-		}
+		c.entriesDepthLimit = depthLimit
 		return nil
 	}
 }

pkg/ads/dist.go

@@ -63,7 +63,7 @@ func adsDistAction(cctx *cli.Context) error {
 		endStr = "head"
 	}
 
-	adCount, err := provClient.Distance(cctx.Context, startCid, endCid)
+	adCount, _, err := provClient.Distance(cctx.Context, startCid, endCid)
 	if err != nil {
 		return err
 	}