Merge pull request #10 from ipfs/initial-iteration

feat: initial functional version

README.md

@@ -3,6 +3,14 @@ bifrost-gateway
 
 > Gateway implementation used by [protocol/bifrost-infra](https://github.com/protocol/bifrost-infra)
 
+
+## Usage
+
+```console
+$ go build
+$ ./bifrost-gateway --help
+```
+
 ## Documentation
 
 - HTTP Gateway implementation: [go-libipf/gateway](https://github.com/ipfs/go-libipfs/tree/main/gateway#documentation)
@@ -50,7 +58,6 @@ graph LR
 - IPFS Backend based on https://strn.network and HTTP client talking to it  via [caboose](https://github.com/filecoin-saturn/caboose) or something else.
 - Functional gaps facilitated by temporary delegation to legacy Kubo RPC `(/api/v0`) infra already used by js-ipfs (or dedicated one).
 
-## Open problem
 
 ## Lead Maintainer
 

blockstore.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/filecoin-saturn/caboose"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+)
+
+func newBlockStore(orchestrator, loggingEndpoint string) (blockstore.Blockstore, error) {
+	oe, err := url.Parse(orchestrator)
+	if err != nil {
+		return nil, err
+	}
+
+	le, err := url.Parse(loggingEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	saturnClient := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				ServerName: "strn.pl",
+			},
+		},
+	}
+
+	return caboose.NewCaboose(&caboose.Config{
+		OrchestratorEndpoint: *oe,
+		OrchestratorClient:   http.DefaultClient,
+
+		LoggingEndpoint: *le,
+		LoggingClient:   http.DefaultClient,
+		LoggingInterval: 5 * time.Second,
+
+		DoValidation: true,
+		PoolRefresh:  5 * time.Minute,
+		Client:       saturnClient,
+	})
+}

gateway.go

@@ -0,0 +1,203 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	gopath "path"
+
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-libipfs/blocks"
+	"github.com/ipfs/go-libipfs/files"
+	"github.com/ipfs/go-merkledag"
+	"github.com/ipfs/go-namesys"
+	"github.com/ipfs/go-namesys/resolve"
+	ipfspath "github.com/ipfs/go-path"
+	"github.com/ipfs/go-path/resolver"
+	"github.com/ipfs/go-unixfs"
+	ufile "github.com/ipfs/go-unixfs/file"
+	uio "github.com/ipfs/go-unixfs/io"
+	"github.com/ipfs/go-unixfsnode"
+	iface "github.com/ipfs/interface-go-ipfs-core"
+	nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys"
+	ifacepath "github.com/ipfs/interface-go-ipfs-core/path"
+	dagpb "github.com/ipld/go-codec-dagpb"
+	"github.com/ipld/go-ipld-prime"
+	"github.com/ipld/go-ipld-prime/node/basicnode"
+	"github.com/ipld/go-ipld-prime/schema"
+	"github.com/libp2p/go-libp2p/core/routing"
+)
+
+type bifrostGateway struct {
+	blockStore   blockstore.Blockstore
+	blockService blockservice.BlockService
+	dagService   format.DAGService
+	resolver     resolver.Resolver
+	namesys      namesys.NameSystem
+	routing      routing.ValueStore
+}
+
+func newBifrostGateway(blockService blockservice.BlockService, routing routing.ValueStore) (*bifrostGateway, error) {
+	// Setup the DAG services, which use the special block store.
+	dagService := merkledag.NewDAGService(blockService)
+
+	// Setup the UnixFS resolver.
+	fetcherConfig := bsfetcher.NewFetcherConfig(blockService)
+	fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
+		if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
+			return tlnkNd.LinkTargetNodePrototype(), nil
+		}
+		return basicnode.Prototype.Any, nil
+	})
+	fetcher := fetcherConfig.WithReifier(unixfsnode.Reify)
+	resolver := resolver.NewBasicResolver(fetcher)
+
+	// Setup name system for DNSLink and IPNS resolution.
+	namesys, err := namesys.NewNameSystem(routing)
+	if err != nil {
+		return nil, err
+	}
+
+	return &bifrostGateway{
+		blockStore:   blockService.Blockstore(),
+		blockService: blockService,
+		dagService:   dagService,
+		resolver:     resolver,
+		namesys:      namesys,
+		routing:      routing,
+	}, nil
+}
+
+func (api *bifrostGateway) GetUnixFsNode(ctx context.Context, p ifacepath.Resolved) (files.Node, error) {
+	nd, err := api.resolveNode(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	return ufile.NewUnixfsFile(ctx, api.dagService, nd)
+}
+
+func (api *bifrostGateway) LsUnixFsDir(ctx context.Context, p ifacepath.Resolved) (<-chan iface.DirEntry, error) {
+	node, err := api.resolveNode(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	dir, err := uio.NewDirectoryFromNode(api.dagService, node)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make(chan iface.DirEntry, uio.DefaultShardWidth)
+
+	go func() {
+		defer close(out)
+		for l := range dir.EnumLinksAsync(ctx) {
+			select {
+			case out <- api.processLink(ctx, l):
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return out, nil
+}
+
+func (api *bifrostGateway) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	return api.blockService.GetBlock(ctx, c)
+}
+
+func (api *bifrostGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
+	return api.routing.GetValue(ctx, "/ipns/"+c.String())
+}
+
+func (api *bifrostGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) {
+	p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1))
+	if err == namesys.ErrResolveRecursion {
+		err = nil
+	}
+	return ifacepath.New(p.String()), err
+}
+
+func (api *bifrostGateway) IsCached(ctx context.Context, p ifacepath.Path) bool {
+	rp, err := api.ResolvePath(ctx, p)
+	if err != nil {
+		return false
+	}
+
+	has, _ := api.blockStore.Has(ctx, rp.Cid())
+	return has
+}
+
+func (api *bifrostGateway) ResolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) {
+	if _, ok := p.(ifacepath.Resolved); ok {
+		return p.(ifacepath.Resolved), nil
+	}
+
+	err := p.IsValid()
+	if err != nil {
+		return nil, err
+	}
+
+	ipath := ipfspath.Path(p.String())
+	if ipath.Segments()[0] == "ipns" {
+		ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if ipath.Segments()[0] != "ipfs" {
+		return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace())
+	}
+
+	node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath)
+	if err != nil {
+		return nil, err
+	}
+
+	root, err := cid.Parse(ipath.Segments()[1])
+	if err != nil {
+		return nil, err
+	}
+
+	return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil
+}
+
+func (api *bifrostGateway) resolveNode(ctx context.Context, p ifacepath.Path) (format.Node, error) {
+	rp, err := api.ResolvePath(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	node, err := api.dagService.Get(ctx, rp.Cid())
+	if err != nil {
+		return nil, fmt.Errorf("get node: %w", err)
+	}
+	return node, nil
+}
+
+func (api *bifrostGateway) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry {
+	if result.Err != nil {
+		return iface.DirEntry{Err: result.Err}
+	}
+
+	link := iface.DirEntry{
+		Name: result.Link.Name,
+		Cid:  result.Link.Cid,
+	}
+
+	switch link.Cid.Type() {
+	case cid.Raw:
+		link.Type = iface.TFile
+		link.Size = result.Link.Size
+	case cid.DagProtobuf:
+		link.Size = result.Link.Size
+	}
+
+	return link
+}

go.mod

@@ -0,0 +1,119 @@
+module github.com/protocol/bifrost-gateway
+
+go 1.19
+
+require (
+	github.com/filecoin-saturn/caboose v0.0.0-20230207140819-1c86f6f71fa3
+	github.com/gogo/protobuf v1.3.2
+	github.com/ipfs/go-blockservice v0.5.0
+	github.com/ipfs/go-cid v0.3.2
+	github.com/ipfs/go-fetcher v1.6.1
+	github.com/ipfs/go-ipfs-blockstore v1.2.0
+	github.com/ipfs/go-ipfs-exchange-offline v0.3.0
+	github.com/ipfs/go-ipld-format v0.4.0
+	github.com/ipfs/go-ipns v0.3.0
+	github.com/ipfs/go-libipfs v0.4.1-0.20230207021459-1a932f7bb3c1
+	github.com/ipfs/go-merkledag v0.9.0
+	github.com/ipfs/go-namesys v0.7.0
+	github.com/ipfs/go-path v0.3.0
+	github.com/ipfs/go-unixfs v0.2.4
+	github.com/ipfs/go-unixfsnode v1.1.2
+	github.com/ipfs/interface-go-ipfs-core v0.10.0
+	github.com/ipld/go-codec-dagpb v1.5.0
+	github.com/ipld/go-ipld-prime v0.19.0
+	github.com/libp2p/go-libp2p v0.23.4
+	github.com/prometheus/client_golang v1.14.0
+	github.com/spf13/cobra v1.6.1
+)
+
+require (
+	github.com/Stebalien/go-bitfield v0.0.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/buraksezer/consistent v0.10.0 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.1 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-block-format v0.1.1 // indirect
+	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
+	github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
+	github.com/ipfs/go-ipfs-files v0.3.0 // indirect
+	github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
+	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.5.1 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-verifcid v0.0.2 // indirect
+	github.com/ipld/go-car v0.5.0 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/libp2p/go-cidranger v1.1.0 // indirect
+	github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
+	github.com/libp2p/go-libp2p-kad-dht v0.19.0 // indirect
+	github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect
+	github.com/libp2p/go-libp2p-record v0.2.0 // indirect
+	github.com/libp2p/go-msgio v0.2.0 // indirect
+	github.com/libp2p/go-netroute v0.2.0 // indirect
+	github.com/libp2p/go-openssl v0.1.0 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-pointer v0.0.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/miekg/dns v1.1.50 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr v0.8.0 // indirect
+	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multibase v0.1.1 // indirect
+	github.com/multiformats/go-multicodec v0.7.0 // indirect
+	github.com/multiformats/go-multihash v0.2.1 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/polydawn/refmt v0.89.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.39.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
+	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
+	github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	go.opentelemetry.io/otel v1.12.0 // indirect
+	go.opentelemetry.io/otel/trace v1.12.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/crypto v0.5.0 // indirect
+	golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 // indirect
+	golang.org/x/mod v0.7.0 // indirect
+	golang.org/x/net v0.5.0 // indirect
+	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/tools v0.5.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	lukechampine.com/blake3 v1.1.7 // indirect
+)