Skip to content

Commit

Permalink
feat: update go-libipfs and switch gateway code to use go-libipfs imp…
Browse files Browse the repository at this point in the history
…lementation
  • Loading branch information
aschmahmann authored and hacdias committed Mar 9, 2023
1 parent 40d7f2f commit 943c471
Show file tree
Hide file tree
Showing 5 changed files with 104 additions and 62 deletions.
154 changes: 98 additions & 56 deletions core/corehttp/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,25 @@ package corehttp

import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"

cid "github.com/ipfs/go-cid"
"github.com/ipfs/go-libipfs/blocks"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
offline "github.com/ipfs/go-ipfs-exchange-offline"
offlineroute "github.com/ipfs/go-ipfs-routing/offline"
"github.com/ipfs/go-libipfs/files"
"github.com/ipfs/go-libipfs/gateway"
"github.com/ipfs/go-namesys"
iface "github.com/ipfs/interface-go-ipfs-core"
options "github.com/ipfs/interface-go-ipfs-core/options"
nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys"
"github.com/ipfs/interface-go-ipfs-core/path"
"github.com/ipfs/kubo/core/node"
"github.com/libp2p/go-libp2p/core/routing"

"github.com/ipfs/go-libipfs/gateway"
options "github.com/ipfs/interface-go-ipfs-core/options"
version "github.com/ipfs/kubo"
config "github.com/ipfs/kubo/config"
core "github.com/ipfs/kubo/core"
Expand Down Expand Up @@ -47,7 +52,7 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
Headers: headers,
}

gwAPI, err := newGatewayAPI(n)
gwAPI, err := newGatewayBackend(n)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -95,7 +100,7 @@ func HostnameOption() ServeOption {
return nil, err
}

gwAPI, err := newGatewayAPI(n)
gwAPI, err := newGatewayBackend(n)
if err != nil {
return nil, err
}
Expand All @@ -118,83 +123,120 @@ func VersionOption() ServeOption {
}
}

type gatewayAPI struct {
ns namesys.NameSystem
api iface.CoreAPI
offlineAPI iface.CoreAPI
}

func newGatewayAPI(n *core.IpfsNode) (*gatewayAPI, error) {
func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
cfg, err := n.Repo.Config()
if err != nil {
return nil, err
}

api, err := coreapi.NewCoreAPI(n, options.Api.FetchBlocks(!cfg.Gateway.NoFetch))
if err != nil {
return nil, err
bserv := n.Blocks
var vsRouting routing.ValueStore = n.Routing
nsys := n.Namesys
if cfg.Gateway.NoFetch {
bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore()))

cs := cfg.Ipns.ResolveCacheSize
if cs == 0 {
cs = node.DefaultIpnsCacheSize
}
if cs < 0 {
return nil, fmt.Errorf("cannot specify negative resolve cache size")
}

vsRouting = offlineroute.NewOfflineRouter(n.Repo.Datastore(), n.RecordValidator)
nsys, err = namesys.NewNameSystem(vsRouting,
namesys.WithDatastore(n.Repo.Datastore()),
namesys.WithDNSResolver(n.DNSResolver),
namesys.WithCache(cs))
if err != nil {
return nil, fmt.Errorf("error constructing namesys: %w", err)
}
}
offlineAPI, err := api.WithOptions(options.Api.Offline(true))

gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
if err != nil {
return nil, err
}
return &offlineGatewayErrWrapper{gwimpl: gw}, nil
}

type offlineGatewayErrWrapper struct {
gwimpl gateway.IPFSBackend
}

func offlineErrWrap(err error) error {
if errors.Is(err, iface.ErrOffline) {
return fmt.Errorf("%s : %w", err.Error(), gateway.ErrServiceUnavailable)
}
return err
}

return &gatewayAPI{
ns: n.Namesys,
api: api,
offlineAPI: offlineAPI,
}, nil
func (o *offlineGatewayErrWrapper) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
md, n, err := o.gwimpl.Get(ctx, path)
err = offlineErrWrap(err)
return md, n, err
}

func (gw *gatewayAPI) GetUnixFsNode(ctx context.Context, pth path.Resolved) (files.Node, error) {
return gw.api.Unixfs().Get(ctx, pth)
func (o *offlineGatewayErrWrapper) GetRange(ctx context.Context, path gateway.ImmutablePath, ranges ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) {
md, n, err := o.gwimpl.GetRange(ctx, path, ranges...)
err = offlineErrWrap(err)
return md, n, err
}

func (gw *gatewayAPI) LsUnixFsDir(ctx context.Context, pth path.Resolved) (<-chan iface.DirEntry, error) {
// Optimization: use Unixfs.Ls without resolving children, but using the
// cumulative DAG size as the file size. This allows for a fast listing
// while keeping a good enough Size field.
return gw.api.Unixfs().Ls(ctx, pth,
options.Unixfs.ResolveChildren(false),
options.Unixfs.UseCumulativeSize(true),
)
func (o *offlineGatewayErrWrapper) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
md, n, err := o.gwimpl.GetAll(ctx, path)
err = offlineErrWrap(err)
return md, n, err
}

func (gw *gatewayAPI) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
r, err := gw.api.Block().Get(ctx, path.IpfsPath(cid))
if err != nil {
return nil, err
}
func (o *offlineGatewayErrWrapper) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) {
md, n, err := o.gwimpl.GetBlock(ctx, path)
err = offlineErrWrap(err)
return md, n, err
}

data, err := io.ReadAll(r)
if err != nil {
return nil, err
}
func (o *offlineGatewayErrWrapper) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
md, n, err := o.gwimpl.Head(ctx, path)
err = offlineErrWrap(err)
return md, n, err
}

return blocks.NewBlockWithCid(data, cid)
func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) {
md, err := o.gwimpl.ResolvePath(ctx, path)
err = offlineErrWrap(err)
return md, err
}

func (gw *gatewayAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
return gw.api.Routing().Get(ctx, "/ipns/"+c.String())
func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
md, data, errCh, err := o.gwimpl.GetCAR(ctx, path)
err = offlineErrWrap(err)
return md, data, errCh, err
}

func (gw *gatewayAPI) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) {
p, err := gw.ns.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1))
if err == namesys.ErrResolveRecursion {
err = nil
}
return path.New(p.String()), err
func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool {
return o.gwimpl.IsCached(ctx, path)
}

func (o *offlineGatewayErrWrapper) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
rec, err := o.gwimpl.GetIPNSRecord(ctx, c)
err = offlineErrWrap(err)
return rec, err
}

func (gw *gatewayAPI) IsCached(ctx context.Context, pth path.Path) bool {
_, err := gw.offlineAPI.Block().Stat(ctx, pth)
return err == nil
func (o *offlineGatewayErrWrapper) ResolveMutable(ctx context.Context, path path.Path) (gateway.ImmutablePath, error) {
imPath, err := o.gwimpl.ResolveMutable(ctx, path)
err = offlineErrWrap(err)
return imPath, err
}

func (gw *gatewayAPI) ResolvePath(ctx context.Context, pth path.Path) (path.Resolved, error) {
return gw.api.ResolvePath(ctx, pth)
func (o *offlineGatewayErrWrapper) GetDNSLinkRecord(ctx context.Context, s string) (path.Path, error) {
p, err := o.gwimpl.GetDNSLinkRecord(ctx, s)
err = offlineErrWrap(err)
return p, err
}

var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil)

var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"}

var subdomainGatewaySpec = &gateway.Specification{
Expand Down
2 changes: 1 addition & 1 deletion docs/examples/kubo-as-a-library/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ go 1.18
replace github.com/ipfs/kubo => ./../../..

require (
github.com/ipfs/go-libipfs v0.6.1
github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b
github.com/ipfs/interface-go-ipfs-core v0.11.0
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
github.com/libp2p/go-libp2p v0.26.2
Expand Down
4 changes: 2 additions & 2 deletions docs/examples/kubo-as-a-library/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -569,8 +569,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2
github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A=
github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24=
github.com/ipfs/go-libipfs v0.6.1 h1:OSO9cm1H3r4OXfP0MP1Q5UhTnhd2fByGl6CVYyz/Rhk=
github.com/ipfs/go-libipfs v0.6.1/go.mod h1:FmhKgxMOQA572TK5DA3MZ5GL44ZqsMHIrkgK4gLn4A8=
github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b h1:8/Y3maFcpkED+e0mMJXHG3BobNYsJxocFamzuhCBoyU=
github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b/go.mod h1:uC9VqJdOglX/UMYH4vX3R64ab0TqEj1qVsT4ncEQTFM=
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ require (
github.com/ipfs/go-ipld-git v0.1.1
github.com/ipfs/go-ipld-legacy v0.1.1
github.com/ipfs/go-ipns v0.3.0
github.com/ipfs/go-libipfs v0.6.1
github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b
github.com/ipfs/go-log v1.0.5
github.com/ipfs/go-log/v2 v2.5.1
github.com/ipfs/go-merkledag v0.9.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -591,8 +591,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2
github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A=
github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24=
github.com/ipfs/go-libipfs v0.6.1 h1:OSO9cm1H3r4OXfP0MP1Q5UhTnhd2fByGl6CVYyz/Rhk=
github.com/ipfs/go-libipfs v0.6.1/go.mod h1:FmhKgxMOQA572TK5DA3MZ5GL44ZqsMHIrkgK4gLn4A8=
github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b h1:8/Y3maFcpkED+e0mMJXHG3BobNYsJxocFamzuhCBoyU=
github.com/ipfs/go-libipfs v0.6.2-0.20230309095205-cde04e59712b/go.mod h1:uC9VqJdOglX/UMYH4vX3R64ab0TqEj1qVsT4ncEQTFM=
github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
Expand Down

0 comments on commit 943c471

Please sign in to comment.