feat: refactor gateway api to operate on higher level semantics (#176)

* refactor gateway api to operate on semantics to closer match user requests
* rename gateway api interface to IPFSBackend
* add blocks gateway implementation to the gateway package
* refactored gateway examples
* add default DNS resolvers
* add default IPLD codecs

---------

Co-authored-by: Henrique Dias <hacdias@gmail.com>
Co-authored-by: Marcin Rataj <lidel@lidel.org>

examples/gateway/car/main.go

@@ -29,7 +29,7 @@ func main() {
 	}
 	defer f.Close()
 
-	gwAPI, err := common.NewBlocksGateway(blockService, nil)
+	gwAPI, err := gateway.NewBlocksGateway(blockService)
 	if err != nil {
 		log.Fatal(err)
 	}

examples/gateway/car/main_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/ipfs/boxo/examples/gateway/common"
+	"github.com/ipfs/boxo/gateway"
 	"github.com/ipld/go-ipld-prime/codec/dagjson"
 	"github.com/ipld/go-ipld-prime/node/basicnode"
 	"github.com/stretchr/testify/assert"
@@ -22,7 +23,7 @@ func newTestServer() (*httptest.Server, io.Closer, error) {
 		return nil, nil, err
 	}
 
-	gateway, err := common.NewBlocksGateway(blockService, nil)
+	gateway, err := gateway.NewBlocksGateway(blockService)
 	if err != nil {
 		_ = f.Close()
 		return nil, nil, err

examples/gateway/common/blocks.go

@@ -1,42 +1,12 @@
 package common
 
 import (
-	"context"
-	"errors"
-	"fmt"
 	"net/http"
-	gopath "path"
 
-	"github.com/ipfs/boxo/blockservice"
-	blockstore "github.com/ipfs/boxo/blockstore"
-	iface "github.com/ipfs/boxo/coreiface"
-	nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
-	ifacepath "github.com/ipfs/boxo/coreiface/path"
-	bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
-	"github.com/ipfs/boxo/files"
 	"github.com/ipfs/boxo/gateway"
-	"github.com/ipfs/boxo/ipld/merkledag"
-	"github.com/ipfs/boxo/ipld/unixfs"
-	ufile "github.com/ipfs/boxo/ipld/unixfs/file"
-	uio "github.com/ipfs/boxo/ipld/unixfs/io"
-	"github.com/ipfs/boxo/namesys"
-	"github.com/ipfs/boxo/namesys/resolve"
-	ipfspath "github.com/ipfs/boxo/path"
-	"github.com/ipfs/boxo/path/resolver"
-	"github.com/ipfs/go-block-format"
-	"github.com/ipfs/go-cid"
-	format "github.com/ipfs/go-ipld-format"
-	"github.com/ipfs/go-unixfsnode"
-	dagpb "github.com/ipld/go-codec-dagpb"
-	"github.com/ipld/go-ipld-prime"
-	"github.com/ipld/go-ipld-prime/node/basicnode"
-	"github.com/ipld/go-ipld-prime/schema"
-	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/libp2p/go-libp2p/core/routing"
-	mc "github.com/multiformats/go-multicodec"
 )
 
-func NewBlocksHandler(gw *BlocksGateway, port int) http.Handler {
+func NewBlocksHandler(gw gateway.IPFSBackend, port int) http.Handler {
 	headers := map[string][]string{}
 	gateway.AddAccessControlHeaders(headers)
 
@@ -50,202 +20,3 @@ func NewBlocksHandler(gw *BlocksGateway, port int) http.Handler {
 	mux.Handle("/ipns/", gwHandler)
 	return mux
 }
-
-type BlocksGateway struct {
-	blockStore   blockstore.Blockstore
-	blockService blockservice.BlockService
-	dagService   format.DAGService
-	resolver     resolver.Resolver
-
-	// Optional routing system to handle /ipns addresses.
-	namesys namesys.NameSystem
-	routing routing.ValueStore
-}
-
-func NewBlocksGateway(blockService blockservice.BlockService, routing routing.ValueStore) (*BlocksGateway, error) {
-	// Setup the DAG services, which use the CAR block store.
-	dagService := merkledag.NewDAGService(blockService)
-
-	// Setup the UnixFS resolver.
-	fetcherConfig := bsfetcher.NewFetcherConfig(blockService)
-	fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
-		if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
-			return tlnkNd.LinkTargetNodePrototype(), nil
-		}
-		return basicnode.Prototype.Any, nil
-	})
-	fetcher := fetcherConfig.WithReifier(unixfsnode.Reify)
-	resolver := resolver.NewBasicResolver(fetcher)
-
-	// Setup a name system so that we are able to resolve /ipns links.
-	var (
-		ns  namesys.NameSystem
-		err error
-	)
-	if routing != nil {
-		ns, err = namesys.NewNameSystem(routing)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return &BlocksGateway{
-		blockStore:   blockService.Blockstore(),
-		blockService: blockService,
-		dagService:   dagService,
-		resolver:     resolver,
-		routing:      routing,
-		namesys:      ns,
-	}, nil
-}
-
-func (api *BlocksGateway) GetUnixFsNode(ctx context.Context, p ifacepath.Resolved) (files.Node, error) {
-	nd, err := api.resolveNode(ctx, p)
-	if err != nil {
-		return nil, err
-	}
-
-	return ufile.NewUnixfsFile(ctx, api.dagService, nd)
-}
-
-func (api *BlocksGateway) LsUnixFsDir(ctx context.Context, p ifacepath.Resolved) (<-chan iface.DirEntry, error) {
-	node, err := api.resolveNode(ctx, p)
-	if err != nil {
-		return nil, err
-	}
-
-	dir, err := uio.NewDirectoryFromNode(api.dagService, node)
-	if err != nil {
-		return nil, err
-	}
-
-	out := make(chan iface.DirEntry, uio.DefaultShardWidth)
-
-	go func() {
-		defer close(out)
-		for l := range dir.EnumLinksAsync(ctx) {
-			select {
-			case out <- api.processLink(ctx, l):
-			case <-ctx.Done():
-				return
-			}
-		}
-	}()
-
-	return out, nil
-}
-
-func (api *BlocksGateway) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
-	return api.blockService.GetBlock(ctx, c)
-}
-
-func (api *BlocksGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
-	if api.routing == nil {
-		return nil, routing.ErrNotSupported
-	}
-
-	// Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful
-	// round trips to the remote routing provider.
-	if mc.Code(c.Type()) != mc.Libp2pKey {
-		return nil, errors.New("provided cid is not an encoded libp2p key")
-	}
-
-	// The value store expects the key itself to be encoded as a multihash.
-	id, err := peer.FromCid(c)
-	if err != nil {
-		return nil, err
-	}
-
-	return api.routing.GetValue(ctx, "/ipns/"+string(id))
-}
-
-func (api *BlocksGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) {
-	if api.namesys != nil {
-		p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1))
-		if err == namesys.ErrResolveRecursion {
-			err = nil
-		}
-		return ifacepath.New(p.String()), err
-	}
-
-	return nil, errors.New("not implemented")
-}
-
-func (api *BlocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool {
-	rp, err := api.ResolvePath(ctx, p)
-	if err != nil {
-		return false
-	}
-
-	has, _ := api.blockStore.Has(ctx, rp.Cid())
-	return has
-}
-
-func (api *BlocksGateway) ResolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) {
-	if _, ok := p.(ifacepath.Resolved); ok {
-		return p.(ifacepath.Resolved), nil
-	}
-
-	err := p.IsValid()
-	if err != nil {
-		return nil, err
-	}
-
-	ipath := ipfspath.Path(p.String())
-	if ipath.Segments()[0] == "ipns" {
-		ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if ipath.Segments()[0] != "ipfs" {
-		return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace())
-	}
-
-	node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath)
-	if err != nil {
-		return nil, err
-	}
-
-	root, err := cid.Parse(ipath.Segments()[1])
-	if err != nil {
-		return nil, err
-	}
-
-	return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil
-}
-
-func (api *BlocksGateway) resolveNode(ctx context.Context, p ifacepath.Path) (format.Node, error) {
-	rp, err := api.ResolvePath(ctx, p)
-	if err != nil {
-		return nil, err
-	}
-
-	node, err := api.dagService.Get(ctx, rp.Cid())
-	if err != nil {
-		return nil, fmt.Errorf("get node: %w", err)
-	}
-	return node, nil
-}
-
-func (api *BlocksGateway) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry {
-	if result.Err != nil {
-		return iface.DirEntry{Err: result.Err}
-	}
-
-	link := iface.DirEntry{
-		Name: result.Link.Name,
-		Cid:  result.Link.Cid,
-	}
-
-	switch link.Cid.Type() {
-	case cid.Raw:
-		link.Type = iface.TFile
-		link.Size = result.Link.Size
-	case cid.DagProtobuf:
-		link.Size = result.Link.Size
-	}
-
-	return link
-}

examples/gateway/proxy/main.go

@@ -27,7 +27,7 @@ func main() {
 	routing := newProxyRouting(*gatewayUrlPtr, nil)
 
 	// Creates the gateway with the block service and the routing.
-	gwAPI, err := common.NewBlocksGateway(blockService, routing)
+	gwAPI, err := gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing))
 	if err != nil {
 		log.Fatal(err)
 	}

examples/gateway/proxy/main_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/ipfs/boxo/blockservice"
 	"github.com/ipfs/boxo/examples/gateway/common"
 	offline "github.com/ipfs/boxo/exchange/offline"
+	"github.com/ipfs/boxo/gateway"
 	"github.com/ipfs/go-block-format"
 	"github.com/stretchr/testify/assert"
 )
@@ -22,12 +23,12 @@ func newProxyGateway(t *testing.T, rs *httptest.Server) *httptest.Server {
 	blockService := blockservice.New(blockStore, offline.Exchange(blockStore))
 	routing := newProxyRouting(rs.URL, nil)
 
-	gateway, err := common.NewBlocksGateway(blockService, routing)
+	gw, err := gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing))
 	if err != nil {
 		t.Error(err)
 	}
 
-	handler := common.NewBlocksHandler(gateway, 0)
+	handler := common.NewBlocksHandler(gw, 0)
 	ts := httptest.NewServer(handler)
 	t.Cleanup(ts.Close)
 

examples/go.mod

@@ -8,9 +8,6 @@ require (
 	github.com/ipfs/go-block-format v0.1.2
 	github.com/ipfs/go-cid v0.4.0
 	github.com/ipfs/go-datastore v0.6.0
-	github.com/ipfs/go-ipld-format v0.4.0
-	github.com/ipfs/go-unixfsnode v1.6.0
-	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/ipld/go-ipld-prime v0.20.0
 	github.com/libp2p/go-libp2p v0.26.3
 	github.com/libp2p/go-libp2p-routing-helpers v0.6.0
@@ -59,12 +56,15 @@ require (
 	github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
 	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
 	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
+	github.com/ipfs/go-ipld-format v0.4.0 // indirect
 	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
 	github.com/ipfs/go-ipns v0.3.0 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
 	github.com/ipfs/go-log/v2 v2.5.1 // indirect
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
+	github.com/ipfs/go-unixfsnode v1.6.0 // indirect
+	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
@@ -73,6 +73,7 @@ require (
 	github.com/koron/go-ssdp v0.0.3 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-cidranger v1.1.0 // indirect
+	github.com/libp2p/go-doh-resolver v0.4.0 // indirect
 	github.com/libp2p/go-flow-metrics v0.1.0 // indirect
 	github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
 	github.com/libp2p/go-libp2p-kad-dht v0.21.1 // indirect

examples/go.sum

@@ -375,6 +375,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c=
 github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic=
+github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+0S7FQqw=
+github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg=
 github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
 github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
 github.com/libp2p/go-libp2p v0.26.3 h1:6g/psubqwdaBqNNoidbRKSTBEYgaOuKBhHl8Q5tO+PM=
@@ -450,6 +452,7 @@ github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU
 github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
 github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU=
 github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs=
+github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ=
 github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A=
 github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk=
 github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=

files/readerfile.go

@@ -18,7 +18,18 @@ type ReaderFile struct {
 }
 
 func NewBytesFile(b []byte) File {
-	return &ReaderFile{"", NewReaderFile(bytes.NewReader(b)), nil, int64(len(b))}
+	return &ReaderFile{"", bytesReaderCloser{bytes.NewReader(b)}, nil, int64(len(b))}
+}
+
+// TODO: Is this the best way to fix this bug?
+// The bug is we want to be an io.ReadSeekCloser, but bytes.NewReader only gives a io.ReadSeeker and io.NopCloser
+// effectively removes the io.Seeker ability from the passed in io.Reader
+type bytesReaderCloser struct {
+	*bytes.Reader
+}
+
+func (b bytesReaderCloser) Close() error {
+	return nil
 }
 
 func NewReaderFile(reader io.Reader) File {