feat: add local index directory

Internally this is still refered to as the piece directory

Co-authored-by: dirkmc <dirkmdev@gmail.com>
Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>

.circleci/config.yml

@@ -1,6 +1,6 @@
 version: 2.1
 orbs:
-  go: gotest/tools@0.0.13
+  go: gotest/tools@0.0.14
 
 executors:
   golang:
@@ -98,6 +98,10 @@ jobs:
         type: string
         default: "./..."
         description: Import paths of packages to be tested.
+      cwd:
+        type: string
+        default: "."
+        description: Directory in which to run the test command
       suite:
         type: string
         default: unit
@@ -110,9 +114,16 @@ jobs:
           command: make boostci
           no_output_timeout: 30m
       - download-params
+      # Note: setup_remote_docker takes about 10s and it's only needed for
+      # local index directory tests so it may be worth refactoring so that it's
+      # only included for local index directory
+      - setup_remote_docker:
+          version: 20.10.18
+          docker_layer_caching: true
       - run:
           name: go test
           command: |
+            cd << parameters.cwd >>
             go test \
               << parameters.go-test-flags >> \
               << parameters.target >>
@@ -310,3 +321,8 @@ workflows:
           name: test-all
           suite: all
           target: "`go list ./... | grep -v boost/itests`"
+
+      - test:
+          name: local index directory
+          suite: all
+          cwd: "./extern/boostd-data"

Makefile

@@ -226,7 +226,7 @@ $(lotus_src_dir):
 	git clone --depth 1 --branch $(lotus_version) https://github.com/filecoin-project/lotus $@
 update/lotus: $(lotus_src_dir)
 	cd $(lotus_src_dir) && git pull
-.PHONY: update/lotus	
+.PHONY: update/lotus
 
 docker/lotus-all-in-one: info/lotus-all-in-one | $(lotus_src_dir)
 	cd $(lotus_src_dir) && $(docker_build_cmd) -f Dockerfile --target lotus-all-in-one \

api/api.go

@@ -6,7 +6,6 @@ import (
 	smtypes "github.com/filecoin-project/boost/storagemarket/types"
 	"github.com/filecoin-project/go-address"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
-	"github.com/filecoin-project/go-fil-markets/piecestore"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -35,6 +34,7 @@ type Boost interface {
 
 	// MethodGroup: Boost
 	BoostIndexerAnnounceAllDeals(ctx context.Context) error                                                                        //perm:admin
+	BoostIndexerListMultihashes(ctx context.Context, proposalCid cid.Cid) ([]multihash.Multihash, error)                           //perm:admin
 	BoostOfflineDealWithData(ctx context.Context, dealUuid uuid.UUID, filePath string) (*ProviderDealRejectionInfo, error)         //perm:admin
 	BoostDeal(ctx context.Context, dealUuid uuid.UUID) (*smtypes.ProviderDealState, error)                                         //perm:admin
 	BoostDealBySignedProposalCid(ctx context.Context, proposalCid cid.Cid) (*smtypes.ProviderDealState, error)                     //perm:admin
@@ -54,6 +54,10 @@ type Boost interface {
 	BlockstoreHas(ctx context.Context, c cid.Cid) (bool, error)    //perm:read
 	BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) //perm:read
 
+	// MethodGroup: PieceDirectory
+	PdBuildIndexForPieceCid(ctx context.Context, piececid cid.Cid) error        //perm:admin
+	PdMarkIndexErrored(ctx context.Context, piececid cid.Cid, err string) error //perm:admin
+
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.
 	RuntimeSubsystems(ctx context.Context) (lapi.MinerSubsystems, error) //perm:read
@@ -73,12 +77,6 @@ type Boost interface {
 	MarketPendingDeals(ctx context.Context) (lapi.PendingDealInfo, error)                                                                                                                //perm:write
 	SectorsRefs(context.Context) (map[string][]lapi.SealedRef, error)                                                                                                                    //perm:read
 
-	PiecesListPieces(ctx context.Context) ([]cid.Cid, error)                                 //perm:read
-	PiecesListCidInfos(ctx context.Context) ([]cid.Cid, error)                               //perm:read
-	PiecesGetPieceInfo(ctx context.Context, pieceCid cid.Cid) (*piecestore.PieceInfo, error) //perm:read
-	PiecesGetCIDInfo(ctx context.Context, payloadCid cid.Cid) (*piecestore.CIDInfo, error)   //perm:read
-	PiecesGetMaxOffset(ctx context.Context, pieceCid cid.Cid) (uint64, error)                //perm:read
-
 	// MethodGroup: Actor
 	ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read
 

cmd/boost/retrieve_cmd.go

@@ -7,11 +7,6 @@ import (
 	"strings"
 	"time"
 
-	flatfs "github.com/ipfs/go-ds-flatfs"
-	levelds "github.com/ipfs/go-ds-leveldb"
-	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	"github.com/mitchellh/go-homedir"
-
 	"github.com/dustin/go-humanize"
 	clinode "github.com/filecoin-project/boost/cli/node"
 	"github.com/filecoin-project/boost/cmd"
@@ -23,6 +18,9 @@ import (
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
+	flatfs "github.com/ipfs/go-ds-flatfs"
+	levelds "github.com/ipfs/go-ds-leveldb"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
 	offline "github.com/ipfs/go-ipfs-exchange-offline"
 	files "github.com/ipfs/go-ipfs-files"
 	"github.com/ipfs/go-merkledag"
@@ -35,6 +33,7 @@ import (
 	"github.com/ipld/go-ipld-prime/traversal/selector"
 	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
 	textselector "github.com/ipld/go-ipld-selector-text-lite"
+	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/term"
 	"golang.org/x/xerrors"

cmd/boostci/utils_cmd.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-
 	"github.com/docker/go-units"
 	"github.com/filecoin-project/boostd-data/shared/cliutil"
 	"github.com/filecoin-project/go-paramfetch"

cmd/boostd/index.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"fmt"
 	bcli "github.com/filecoin-project/boost/cli"
 	lcli "github.com/filecoin-project/lotus/cli"
+	"github.com/ipfs/go-cid"
 	"github.com/urfave/cli/v2"
 )
 
@@ -11,6 +13,7 @@ var indexProvCmd = &cli.Command{
 	Usage: "Manage the index provider on Boost",
 	Subcommands: []*cli.Command{
 		indexProvAnnounceAllCmd,
+		indexProvListMultihashesCmd,
 	},
 }
 
@@ -21,13 +24,51 @@ var indexProvAnnounceAllCmd = &cli.Command{
 	Action: func(cctx *cli.Context) error {
 		ctx := lcli.ReqContext(cctx)
 
-		// announce markets and boost deals
+		// get boost api
 		napi, closer, err := bcli.GetBoostAPI(cctx)
 		if err != nil {
 			return err
 		}
 		defer closer()
 
+		// announce markets and boost deals
 		return napi.BoostIndexerAnnounceAllDeals(ctx)
 	},
 }
+
+var indexProvListMultihashesCmd = &cli.Command{
+	Name:      "list-multihashes",
+	Usage:     "list-multihashes <proposal cid>",
+	UsageText: "List multihashes for a deal by proposal cid",
+	Action: func(cctx *cli.Context) error {
+		if cctx.NArg() != 1 {
+			return fmt.Errorf("must supply proposal cid")
+		}
+
+		propCid, err := cid.Parse(cctx.Args().First())
+		if err != nil {
+			return fmt.Errorf("parsing proposal cid %s: %w", cctx.Args().First(), err)
+		}
+
+		ctx := lcli.ReqContext(cctx)
+
+		// get boost api
+		napi, closer, err := bcli.GetBoostAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		// get list of multihashes
+		mhs, err := napi.BoostIndexerListMultihashes(ctx, propCid)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Found %d multihashes for deal with proposal cid %s:\n", len(mhs), propCid)
+		for _, mh := range mhs {
+			fmt.Println("  " + mh.String())
+		}
+		return nil
+	},
+}

cmd/boostd/main.go

@@ -3,12 +3,11 @@ package main
 import (
 	"os"
 
+	"github.com/filecoin-project/boost/build"
 	"github.com/filecoin-project/boost/cmd"
+	"github.com/filecoin-project/boostd-data/shared/cliutil"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/urfave/cli/v2"
-
-	"github.com/filecoin-project/boost/build"
-	"github.com/filecoin-project/boostd-data/shared/cliutil"
 )
 
 var log = logging.Logger("boostd")
@@ -48,8 +47,8 @@ func main() {
 			importDataCmd,
 			logCmd,
 			dagstoreCmd,
-			piecesCmd,
 			netCmd,
+			pieceDirCmd,
 		},
 	}
 	app.Setup()
@@ -66,6 +65,7 @@ func before(cctx *cli.Context) error {
 	_ = logging.SetLogLevel("modules", "INFO")
 	_ = logging.SetLogLevel("cfg", "INFO")
 	_ = logging.SetLogLevel("boost-storage-deal", "INFO")
+	_ = logging.SetLogLevel("piecedir", "INFO")
 	_ = logging.SetLogLevel("index-provider-wrapper", "INFO")
 
 	if cliutil.IsVeryVerbose {
@@ -78,6 +78,8 @@ func before(cctx *cli.Context) error {
 		_ = logging.SetLogLevel("boost-migrator", "DEBUG")
 		_ = logging.SetLogLevel("dagstore", "DEBUG")
 		_ = logging.SetLogLevel("migrator", "DEBUG")
+		_ = logging.SetLogLevel("piecedir", "DEBUG")
+		_ = logging.SetLogLevel("fxlog", "DEBUG")
 	}
 
 	return nil