From dce7280eb277bb980a3a3139bd5bbd4941a56e4a Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 6 Mar 2023 14:32:48 -0500 Subject: [PATCH 01/27] feat: update go-libipfs and switch to use its blocks-gateway implementation --- gateway.go | 218 ---------------------------------------------------- go.mod | 27 +++---- go.sum | 37 ++++----- handlers.go | 2 +- 4 files changed, 30 insertions(+), 254 deletions(-) delete mode 100644 gateway.go diff --git a/gateway.go b/gateway.go deleted file mode 100644 index 5ba40c2..0000000 --- a/gateway.go +++ /dev/null @@ -1,218 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - gopath "path" - - "github.com/ipfs/go-blockservice" - "github.com/ipfs/go-cid" - bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice" - blockstore "github.com/ipfs/go-ipfs-blockstore" - format "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-libipfs/blocks" - "github.com/ipfs/go-libipfs/files" - "github.com/ipfs/go-merkledag" - "github.com/ipfs/go-namesys" - "github.com/ipfs/go-namesys/resolve" - ipfspath "github.com/ipfs/go-path" - "github.com/ipfs/go-path/resolver" - "github.com/ipfs/go-unixfs" - ufile "github.com/ipfs/go-unixfs/file" - uio "github.com/ipfs/go-unixfs/io" - "github.com/ipfs/go-unixfsnode" - iface "github.com/ipfs/interface-go-ipfs-core" - nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" - ifacepath "github.com/ipfs/interface-go-ipfs-core/path" - dagpb "github.com/ipld/go-codec-dagpb" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/node/basicnode" - "github.com/ipld/go-ipld-prime/schema" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/routing" - mc "github.com/multiformats/go-multicodec" -) - -type bifrostGateway struct { - blockStore blockstore.Blockstore - blockService blockservice.BlockService - dagService format.DAGService - resolver resolver.Resolver - namesys namesys.NameSystem - routing routing.ValueStore -} - -func newBifrostGateway(blockService blockservice.BlockService, routing routing.ValueStore) (*bifrostGateway, error) { - // Setup the DAG services, which use the special block store. - dagService := merkledag.NewDAGService(blockService) - - // Setup the UnixFS resolver. - fetcherConfig := bsfetcher.NewFetcherConfig(blockService) - fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) - resolver := resolver.NewBasicResolver(fetcher) - - // Setup name system for DNSLink and IPNS resolution. - namesys, err := namesys.NewNameSystem(routing) - if err != nil { - return nil, err - } - - return &bifrostGateway{ - blockStore: blockService.Blockstore(), - blockService: blockService, - dagService: dagService, - resolver: resolver, - namesys: namesys, - routing: routing, - }, nil -} - -func (api *bifrostGateway) GetUnixFsNode(ctx context.Context, p ifacepath.Resolved) (files.Node, error) { - nd, err := api.resolveNode(ctx, p) - if err != nil { - return nil, err - } - - return ufile.NewUnixfsFile(ctx, api.dagService, nd) -} - -func (api *bifrostGateway) LsUnixFsDir(ctx context.Context, p ifacepath.Resolved) (<-chan iface.DirEntry, error) { - node, err := api.resolveNode(ctx, p) - if err != nil { - return nil, err - } - - dir, err := uio.NewDirectoryFromNode(api.dagService, node) - if err != nil { - return nil, err - } - - out := make(chan iface.DirEntry, uio.DefaultShardWidth) - - go func() { - defer close(out) - for l := range dir.EnumLinksAsync(ctx) { - select { - case out <- api.processLink(ctx, l): - case <-ctx.Done(): - return - } - } - }() - - return out, nil -} - -func (api *bifrostGateway) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - return api.blockService.GetBlock(ctx, c) -} - -func (api *bifrostGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { - // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful - // round trips to the remote routing provider. - if mc.Code(c.Type()) != mc.Libp2pKey { - return nil, errors.New("provided cid is not an encoded libp2p key") - } - - // The value store expects the key itself to be multihash. - id, err := peer.FromCid(c) - if err != nil { - return nil, err - } - - return api.routing.GetValue(ctx, "/ipns/"+string(id)) -} - -func (api *bifrostGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { - p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) - if err == namesys.ErrResolveRecursion { - err = nil - } - return ifacepath.New(p.String()), err -} - -func (api *bifrostGateway) IsCached(ctx context.Context, p ifacepath.Path) bool { - rp, err := api.ResolvePath(ctx, p) - if err != nil { - return false - } - - has, _ := api.blockStore.Has(ctx, rp.Cid()) - return has -} - -func (api *bifrostGateway) ResolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) { - if _, ok := p.(ifacepath.Resolved); ok { - return p.(ifacepath.Resolved), nil - } - - err := p.IsValid() - if err != nil { - return nil, err - } - - ipath := ipfspath.Path(p.String()) - if ipath.Segments()[0] == "ipns" { - ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) - if err != nil { - return nil, err - } - } - - if ipath.Segments()[0] != "ipfs" { - return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) - } - - node, rest, err := api.resolver.ResolveToLastNode(ctx, ipath) - if err != nil { - return nil, err - } - - root, err := cid.Parse(ipath.Segments()[1]) - if err != nil { - return nil, err - } - - return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil -} - -func (api *bifrostGateway) resolveNode(ctx context.Context, p ifacepath.Path) (format.Node, error) { - rp, err := api.ResolvePath(ctx, p) - if err != nil { - return nil, err - } - - node, err := api.dagService.Get(ctx, rp.Cid()) - if err != nil { - return nil, fmt.Errorf("get node: %w", err) - } - return node, nil -} - -func (api *bifrostGateway) processLink(ctx context.Context, result unixfs.LinkResult) iface.DirEntry { - if result.Err != nil { - return iface.DirEntry{Err: result.Err} - } - - link := iface.DirEntry{ - Name: result.Link.Name, - Cid: result.Link.Cid, - } - - switch link.Cid.Type() { - case cid.Raw: - link.Type = iface.TFile - link.Size = result.Link.Size - case cid.DagProtobuf: - link.Size = result.Link.Size - } - - return link -} diff --git a/go.mod b/go.mod index 1ce1fab..6e12fbe 100644 --- a/go.mod +++ b/go.mod @@ -7,24 +7,15 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/ipfs/go-blockservice v0.5.0 - github.com/ipfs/go-cid v0.3.2 - github.com/ipfs/go-fetcher v1.6.1 + github.com/ipfs/go-cid v0.4.0 github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-ipns v0.3.0 - github.com/ipfs/go-libipfs v0.6.1-0.20230306084414-15f2131fe190 + github.com/ipfs/go-libipfs v0.7.1-0.20230321154728-1c4716f9e834 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipfs/go-merkledag v0.9.0 - github.com/ipfs/go-namesys v0.7.0 - github.com/ipfs/go-path v0.3.1 - github.com/ipfs/go-unixfs v0.3.1 - github.com/ipfs/go-unixfsnode v1.5.1 github.com/ipfs/interface-go-ipfs-core v0.11.1 - github.com/ipld/go-codec-dagpb v1.5.0 - github.com/ipld/go-ipld-prime v0.19.0 github.com/libp2p/go-libp2p v0.25.1 - github.com/multiformats/go-multicodec v0.7.0 github.com/prometheus/client_golang v1.14.0 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 github.com/spf13/cobra v1.6.1 @@ -54,28 +45,37 @@ require ( github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/influxdata/tdigest v0.0.1 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/go-bitfield v1.0.0 // indirect + github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-block-format v0.1.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect + github.com/ipfs/go-fetcher v1.6.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect - github.com/ipfs/go-ipfs-files v0.3.0 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect github.com/ipfs/go-ipld-legacy v0.1.1 // indirect github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipfs/go-merkledag v0.9.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect + github.com/ipfs/go-namesys v0.7.0 // indirect + github.com/ipfs/go-path v0.3.1 // indirect + github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa // indirect + github.com/ipfs/go-unixfsnode v1.5.2 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect github.com/ipld/go-car v0.5.0 // indirect + github.com/ipld/go-codec-dagpb v1.5.0 // indirect + github.com/ipld/go-ipld-prime v0.19.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect + github.com/libp2p/go-doh-resolver v0.4.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect github.com/libp2p/go-libp2p-kad-dht v0.21.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect + github.com/libp2p/go-libp2p-routing-helpers v0.4.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect github.com/mattn/go-isatty v0.0.17 // indirect @@ -88,6 +88,7 @@ require ( github.com/multiformats/go-multiaddr v0.8.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multibase v0.1.1 // indirect + github.com/multiformats/go-multicodec v0.7.0 // indirect github.com/multiformats/go-multihash v0.2.1 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect diff --git a/go.sum b/go.sum index 7265128..499036a 100644 --- a/go.sum +++ b/go.sum @@ -287,8 +287,8 @@ github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/Z github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ= -github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= +github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= +github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -305,8 +305,8 @@ github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= +github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA= +github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= @@ -333,7 +333,6 @@ github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxp github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= -github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -345,11 +344,7 @@ github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= -github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= -github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsbGxQ= -github.com/ipfs/go-ipfs-files v0.3.0/go.mod h1:xAUtYMwB+iu/dtf6+muHNSFQCJG2dSiStR2P6sn9tIM= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= -github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-redirects-file v0.1.1 h1:Io++k0Vf/wK+tfnhEh63Yte1oQK5VGT2hIEYpD0Rzx8= @@ -359,7 +354,6 @@ github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9 github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= @@ -367,13 +361,12 @@ github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ= github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc= github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= -github.com/ipfs/go-libipfs v0.6.1-0.20230306084414-15f2131fe190 h1:uLCIMrKU1/SlZVDrTNfX3p6y4hv+cSZn51xiKU3glV8= -github.com/ipfs/go-libipfs v0.6.1-0.20230306084414-15f2131fe190/go.mod h1:FmhKgxMOQA572TK5DA3MZ5GL44ZqsMHIrkgK4gLn4A8= +github.com/ipfs/go-libipfs v0.7.1-0.20230321154728-1c4716f9e834 h1:uTC+XqhuZZw4R23E5auJzB1M1Sqhm3SeSeazit4ujQk= +github.com/ipfs/go-libipfs v0.7.1-0.20230321154728-1c4716f9e834/go.mod h1:4PFL5Fz4pJiI9yFeQg9B9JZqGabCKyEqg+82HF3RW1k= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= @@ -388,7 +381,6 @@ github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Ax github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= -github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4= github.com/ipfs/go-merkledag v0.9.0 h1:DFC8qZ96Dz1hMT7dtIpcY524eFFDiEWAF8hNJHWW2pk= github.com/ipfs/go-merkledag v0.9.0/go.mod h1:bPHqkHt5OZ0p1n3iqPeDiw2jIBkjAytRjS3WSBwjq90= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= @@ -399,10 +391,10 @@ github.com/ipfs/go-path v0.3.1 h1:wkeaCWE/NTuuPGlEkLTsED5UkzfKYZpxaFFPgk8ZVLE= github.com/ipfs/go-path v0.3.1/go.mod h1:eNLsxJEEMxn/CDzUJ6wuNl+6No6tEUhOZcPKsZsYX0E= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= -github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8= -github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o= -github.com/ipfs/go-unixfsnode v1.5.1 h1:JcR3t5C2nM1V7PMzhJ/Qmo19NkoFIKweDSZyDx+CjkI= -github.com/ipfs/go-unixfsnode v1.5.1/go.mod h1:ed79DaG9IEuZITJVQn4U6MZDftv6I3ygUBLPfhEbHvk= +github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa h1:X8DPpsI3xvdsNxrsHi+ji39rjIvfPna3+XD+iQehbNQ= +github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= +github.com/ipfs/go-unixfsnode v1.5.2 h1:CvsiTt58W2uR5dD8bqQv+aAY0c1qolmXmSyNbPHYiew= +github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= @@ -411,7 +403,6 @@ github.com/ipfs/interface-go-ipfs-core v0.11.1/go.mod h1:xmnoccUXY7N/Q8AIx0vFqgW github.com/ipld/go-car v0.5.0 h1:kcCEa3CvYMs0iE5BzD5sV7O2EwMiCIp3uF8tA6APQT8= github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE= github.com/ipld/go-car/v2 v2.6.0 h1:UTJmJ99nxgYpPueoaZ1GrFQtUDe9QnVLlHYTNDSDb90= -github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-codec-dagpb v1.5.0 h1:RspDRdsJpLfgCI0ONhTAnbHdySGD4t+LHSPK4X1+R0k= github.com/ipld/go-codec-dagpb v1.5.0/go.mod h1:0yRIutEFD8o1DGVqw4RSHh+BUTlJA9XWldxaaWR/o4g= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= @@ -484,6 +475,8 @@ github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0 github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-conn-security-multistream v0.2.0/go.mod h1:hZN4MjlNetKD3Rq5Jb/P5ohUnFLNzEAR4DLSzpn2QLU= github.com/libp2p/go-conn-security-multistream v0.2.1/go.mod h1:cR1d8gA0Hr59Fj6NhaTpFhJZrjSYuNmhpT2r25zYR70= +github.com/libp2p/go-doh-resolver v0.4.0 h1:gUBa1f1XsPwtpE1du0O+nnZCUqtG7oYi7Bb+0S7FQqw= +github.com/libp2p/go-doh-resolver v0.4.0/go.mod h1:v1/jwsFusgsWIGX/c6vCRrnJ60x7bhTiq/fs2qt0cAg= github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-eventbus v0.2.1/go.mod h1:jc2S4SoEVPP48H9Wpzm5aiGwUCBMfGhVhhBjyhhCJs8= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= @@ -562,6 +555,8 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0 h1:b7y4aixQ7AwbqYfcOQ6wTw8DQvuRZeTAA0Od3YYN5yc= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= @@ -738,6 +733,7 @@ github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6 github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= +github.com/multiformats/go-multiaddr-dns v0.3.0/go.mod h1:mNzQ4eTGDg0ll1N9jKPOUogZPoJ30W8a7zk66FQPpdQ= github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= @@ -999,7 +995,6 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= -github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= @@ -1172,7 +1167,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1311,7 +1305,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/handlers.go b/handlers.go index 3bfe0b4..2b097c4 100644 --- a/handlers.go +++ b/handlers.go @@ -63,7 +63,7 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC routing := newProxyRouting(kuboRPC, cdns) // Creates the gateway with the block service and the routing. - gwAPI, err := newBifrostGateway(blockService, routing) + gwAPI, err := gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing)) if err != nil { return nil, err } From 874b4dc7b989900017cb84db6ab0d7867a949d2a Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 15 Mar 2023 07:55:45 -0400 Subject: [PATCH 02/27] heavily WIP: add a backend we can test against --- backend/handlers.go | 294 ++++++++++++++++++ backend/main.go | 158 ++++++++++ backend/version.go | 39 +++ go.sum | 83 ++++- handlers.go | 3 +- .../blockstore_cache.go | 15 +- main.go | 3 +- 7 files changed, 584 insertions(+), 11 deletions(-) create mode 100644 backend/handlers.go create mode 100644 backend/main.go create mode 100644 backend/version.go rename blockstore_cache.go => lib/blockstore_cache.go (93%) diff --git a/backend/handlers.go b/backend/handlers.go new file mode 100644 index 0000000..68855f6 --- /dev/null +++ b/backend/handlers.go @@ -0,0 +1,294 @@ +package main + +import ( + "bytes" + "context" + "fmt" + unixfile "github.com/ipfs/go-unixfs/file" + "github.com/ipld/go-car" + "github.com/ipld/go-car/util" + "io" + "net/http" + "os" + "runtime/debug" + "strconv" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + ipath "github.com/ipfs/interface-go-ipfs-core/path" +) + +func makeMetricsHandler(port int) (*http.Server, error) { + mux := http.NewServeMux() + + gatherers := prometheus.Gatherers{ + prometheus.DefaultGatherer, + } + options := promhttp.HandlerOpts{} + mux.Handle("/debug/metrics/prometheus", promhttp.HandlerFor(gatherers, options)) + + return &http.Server{ + Handler: mux, + Addr: ":" + strconv.Itoa(port), + }, nil +} + +func withRequestLogger(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + goLog.Infow(r.Method, "url", r.URL, "host", r.Host) + // TODO: if debug is enabled, show more? goLog.Infow("request received", "url", r.URL, "host", r.Host, "method", r.Method, "ua", r.UserAgent(), "referer", r.Referer()) + next.ServeHTTP(w, r) + }) +} + +var noModtime = time.Unix(0, 0) + +func makeGatewayCARHandler(bsrv blockservice.BlockService, port int) (*http.Server, error) { + mux := http.NewServeMux() + mux.HandleFunc("/ipfs/", func(w http.ResponseWriter, r *http.Request) { + // the hour is a hard fallback, we don't expect it to happen, but just in case + ctx, cancel := context.WithTimeout(r.Context(), time.Hour) + defer cancel() + r = r.WithContext(ctx) + + defer func() { + if r := recover(); r != nil { + goLog.Error("A panic occurred in the gateway handler!") + goLog.Error(r) + debug.PrintStack() + } + }() + + if r.Method != http.MethodGet { + w.Header().Add("Allow", http.MethodGet) + + errmsg := "Method " + r.Method + " not allowed" + http.Error(w, errmsg, http.StatusMethodNotAllowed) + return + } + + isCar := false + if formatParam := r.URL.Query().Get("format"); formatParam != "" { + isCar = formatParam == "car" + if !isCar { + http.Error(w, "only raw format supported", http.StatusBadRequest) + return + } + } else { + for _, header := range r.Header.Values("Accept") { + for _, value := range strings.Split(header, ",") { + accept := strings.TrimSpace(value) + if accept == "application/vnd.ipld.car" { + isCar = true + break + } + } + } + } + if !isCar { + http.Error(w, "only car format supported", http.StatusBadRequest) + return + } + + contentPath := ipath.New(r.URL.Path) + if contentPath.Mutable() { + http.Error(w, "only immutable block requests supported", http.StatusBadRequest) + return + } else if contentPath.Namespace() != "ipfs" { + http.Error(w, "only the ipfs names is supported", http.StatusBadRequest) + return + } + + carStream, err := simpleSelectorToCar(contentPath) + if err != nil { + http.Error(w, "only the ipfs names is supported", http.StatusBadRequest) + return + } + + const immutableCacheControl = "public, max-age=29030400, immutable" + // immutable! CACHE ALL THE THINGS, FOREVER! wolololol + w.Header().Set("Cache-Control", immutableCacheControl) + + // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) + + io.Copy(w, carStream) + return + }) + + // Creates metrics handler for total response size. Matches the same metrics + // from Kubo: + // https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152 + sum := prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: "ipfs", + Subsystem: "http", + Name: "response_size_bytes", + Help: "The HTTP response sizes in bytes.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, nil) + err := prometheus.Register(sum) + if err != nil { + return nil, err + } + + // Construct the HTTP handler for the gateway. + handler := promhttp.InstrumentHandlerResponseSize(sum, mux) + + // Add logging + handler = withRequestLogger(handler) + + return &http.Server{ + Handler: handler, + Addr: ":" + strconv.Itoa(port), + }, nil +} + +func simpleSelectorToCar(ipfsPath ipath.Path) (io.ReadCloser, error) { + pathSegs := strings.Split(ipfsPath.String(), "/") + if len(pathSegs) < 3 || !(pathSegs[0] == "" && pathSegs[1] == "ipfs") { + return nil, fmt.Errorf("invalid path") + } + pathSegs = pathSegs[2:] + rootCidStr := pathSegs[0] + rootCid, err := cid.Decode(rootCidStr) + if err != nil { + return nil, err + } + + r, w := io.Pipe() + // Setup header for the output car + err = car.WriteHeader(&car.CarHeader{ + Roots: []cid.Cid{rootCid}, + Version: 1, + }, w) + if err != nil { + return nil, fmt.Errorf("writing car header: %w", err) + } + + go func() { + defer w.Close() + remainingPath := pathSegs[1:] + unixfile.NewUnixfsFile() + + err = util.LdWrite(os.Stdout, block.Cid().Bytes(), block.RawData()) // write to the output car + if err != nil { + return fmt.Errorf("writing to output car: %w", err) + } + }() + _ = rootCid + return r, nil +} + +func makeGatewayBlockHandler(bsrv blockservice.BlockService, port int) (*http.Server, error) { + mux := http.NewServeMux() + mux.HandleFunc("/ipfs/", func(w http.ResponseWriter, r *http.Request) { + // the hour is a hard fallback, we don't expect it to happen, but just in case + ctx, cancel := context.WithTimeout(r.Context(), time.Hour) + defer cancel() + r = r.WithContext(ctx) + + defer func() { + if r := recover(); r != nil { + goLog.Error("A panic occurred in the gateway handler!") + goLog.Error(r) + debug.PrintStack() + } + }() + + if r.Method != http.MethodGet { + w.Header().Add("Allow", http.MethodGet) + + errmsg := "Method " + r.Method + " not allowed" + http.Error(w, errmsg, http.StatusMethodNotAllowed) + return + } + + isBlock := false + if formatParam := r.URL.Query().Get("format"); formatParam != "" { + isBlock = formatParam == "raw" + if !isBlock { + http.Error(w, "only raw format supported", http.StatusBadRequest) + return + } + } else { + for _, header := range r.Header.Values("Accept") { + for _, value := range strings.Split(header, ",") { + accept := strings.TrimSpace(value) + if accept == "application/vnd.ipld.raw" { + isBlock = true + break + } + } + } + } + if !isBlock { + http.Error(w, "only raw format supported", http.StatusBadRequest) + return + } + + contentPath := ipath.New(r.URL.Path) + if contentPath.Mutable() { + http.Error(w, "only immutable block requests supported", http.StatusBadRequest) + return + } else if contentPath.Namespace() != "ipfs" { + http.Error(w, "only the ipfs names is supported", http.StatusBadRequest) + return + } + + strComps := strings.Split(strings.TrimRight(contentPath.String(), "/"), "/") + if len(strComps) != 3 { + http.Error(w, "requests must be for single raw blocks", http.StatusBadRequest) + return + } + c, err := cid.Decode(strComps[2]) + if err != nil { + http.Error(w, fmt.Sprintf("not a valid cid %s", strComps[2]), http.StatusBadRequest) + return + } + + blk, err := bsrv.GetBlock(r.Context(), c) + if err != nil { + http.Error(w, fmt.Sprintf("could not get cid %s", c), http.StatusInternalServerError) + return + } + + const immutableCacheControl = "public, max-age=29030400, immutable" + // immutable! CACHE ALL THE THINGS, FOREVER! wolololol + w.Header().Set("Cache-Control", immutableCacheControl) + + // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) + + http.ServeContent(w, r, c.String()+".bin", noModtime, bytes.NewReader(blk.RawData())) + return + }) + + // Creates metrics handler for total response size. Matches the same metrics + // from Kubo: + // https://github.com/ipfs/kubo/blob/e550d9e4761ea394357c413c02ade142c0dea88c/core/corehttp/metrics.go#L79-L152 + sum := prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: "ipfs", + Subsystem: "http", + Name: "response_size_bytes", + Help: "The HTTP response sizes in bytes.", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + }, nil) + err := prometheus.Register(sum) + if err != nil { + return nil, err + } + + // Construct the HTTP handler for the gateway. + handler := promhttp.InstrumentHandlerResponseSize(sum, mux) + + // Add logging + handler = withRequestLogger(handler) + + return &http.Server{ + Handler: handler, + Addr: ":" + strconv.Itoa(port), + }, nil +} diff --git a/backend/main.go b/backend/main.go new file mode 100644 index 0000000..35276a8 --- /dev/null +++ b/backend/main.go @@ -0,0 +1,158 @@ +package main + +import ( + "errors" + "fmt" + "github.com/ipfs/bifrost-gateway/lib" + "github.com/ipfs/go-blockservice" + offline "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipfs/go-libipfs/bitswap/client" + "github.com/ipfs/go-libipfs/bitswap/network" + golog "github.com/ipfs/go-log/v2" + carbs "github.com/ipld/go-car/v2/blockstore" + "github.com/libp2p/go-libp2p" + "github.com/spf13/cobra" + "log" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "sync" +) + +var goLog = golog.Logger("bifrost-gateway-backend") + +func main() { + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func init() { + rootCmd.Flags().Int("gateway-port", 8081, "gateway port") + rootCmd.Flags().Int("metrics-port", 8041, "metrics port") + rootCmd.Flags().String("car-blockstore", "", "a CAR file to use for serving data instead of network requests") +} + +var rootCmd = &cobra.Command{ + Use: name, + Version: version, + CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, + Short: "IPFS Gateway backend implementation for https://github.com/protocol/bifrost-infra", + RunE: func(cmd *cobra.Command, args []string) error { + // Get flags. + gatewayPort, _ := cmd.Flags().GetInt("gateway-port") + metricsPort, _ := cmd.Flags().GetInt("metrics-port") + carbsLocation, _ := cmd.Flags().GetString("car-blockstore") + + var bsrv blockservice.BlockService + if carbsLocation == "" { + bs, err := carbs.OpenReadOnly(carbsLocation) + if err != nil { + return err + } + bsrv = blockservice.New(bs, offline.Exchange(bs)) + } else { + //blockCacheSize, err := getEnvInt(EnvBlockCacheSize, lib.DefaultCacheBlockStoreSize) + //if err != nil { + // return err + //} + blockCacheSize := lib.DefaultCacheBlockStoreSize + + bs, err := lib.NewCacheBlockStore(blockCacheSize) + if err != nil { + return err + } + + h, err := libp2p.New() + if err != nil { + return err + } + n := network.NewFromIpfsHost(h, nil) + bsc := client.New(cmd.Context(), n, bs) + n.Start(bsc) + defer n.Stop() + + bsrv = blockservice.New(bs, bsc) + } + + log.Printf("Starting %s %s", name, version) + + gatewaySrv, err := makeGatewayBlockHandler(bsrv, gatewayPort) + if err != nil { + return err + } + + metricsSrv, err := makeMetricsHandler(metricsPort) + if err != nil { + return err + } + + quit := make(chan os.Signal, 1) + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + //log.Printf("%s: %d", EnvBlockCacheSize, blockCacheSize) + log.Printf("Path gateway listening on http://127.0.0.1:%d", gatewayPort) + log.Printf(" Smoke test (JPG): http://127.0.0.1:%d/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi?format=raw", gatewayPort) + log.Printf("Subdomain gateway configured on dweb.link and http://localhost:%d", gatewayPort) + log.Printf(" Smoke test (Subdomain+DNSLink+UnixFS+HAMT): http://localhost:%d/ipns/en.wikipedia-on-ipfs.org/wiki/", gatewayPort) + err := gatewaySrv.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Printf("Failed to start gateway: %s", err) + quit <- os.Interrupt + } + }() + + go func() { + defer wg.Done() + log.Printf("Metrics exposed at http://127.0.0.1:%d/debug/metrics/prometheus", metricsPort) + err := metricsSrv.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Printf("Failed to start metrics: %s", err) + quit <- os.Interrupt + } + }() + + signal.Notify(quit, os.Interrupt) + <-quit + log.Printf("Closing servers...") + go gatewaySrv.Close() + go metricsSrv.Close() + wg.Wait() + return nil + }, +} + +func getEnv(key, defaultValue string) string { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + return value +} + +func getEnvs(key, defaultValue string) []string { + value := os.Getenv(key) + if value == "" { + if defaultValue == "" { + return []string{} + } + value = defaultValue + } + value = strings.TrimSpace(value) + return strings.Split(value, ",") +} + +func getEnvInt(key string, defaultValue int) (int, error) { + value := os.Getenv(key) + if value == "" { + return defaultValue, nil + } + return strconv.Atoi(value) +} diff --git a/backend/version.go b/backend/version.go new file mode 100644 index 0000000..ba6d080 --- /dev/null +++ b/backend/version.go @@ -0,0 +1,39 @@ +package main + +import ( + "runtime/debug" + "time" +) + +var name = "bifrost-gateway-backend" +var version = buildVersion() +var userAgent = name + "/" + version + +func buildVersion() string { + var revision string + var day string + var dirty bool + + info, ok := debug.ReadBuildInfo() + if !ok { + return "dev-build" + } + for _, kv := range info.Settings { + switch kv.Key { + case "vcs.revision": + revision = kv.Value[:7] + case "vcs.time": + t, _ := time.Parse(time.RFC3339, kv.Value) + day = t.UTC().Format("2006-01-02") + case "vcs.modified": + dirty = kv.Value == "true" + } + } + if dirty { + revision += "-dirty" + } + if revision != "" { + return day + "-" + revision + } + return "dev-build" +} diff --git a/go.sum b/go.sum index 499036a..631626e 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,7 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -71,23 +72,28 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= +github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d h1:t5Wuyh53qYyg9eqn4BbnlIT+vmhyww0TatL+zT3uWgI= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= @@ -113,7 +119,9 @@ github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70d github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -121,7 +129,9 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= +github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -144,6 +154,10 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= +github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= @@ -158,11 +172,28 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= +github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -183,6 +214,7 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -208,6 +240,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -221,6 +254,7 @@ github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20221203041831-ce31453925ec h1:fR20TYVVwhK4O7r7y+McjRYyaTH6/vjwJOajE+XhlzM= +github.com/google/pprof v0.0.0-20221203041831-ce31453925ec/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -238,6 +272,7 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -278,6 +313,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= @@ -344,6 +380,7 @@ github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= +github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= @@ -403,12 +440,14 @@ github.com/ipfs/interface-go-ipfs-core v0.11.1/go.mod h1:xmnoccUXY7N/Q8AIx0vFqgW github.com/ipld/go-car v0.5.0 h1:kcCEa3CvYMs0iE5BzD5sV7O2EwMiCIp3uF8tA6APQT8= github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE= github.com/ipld/go-car/v2 v2.6.0 h1:UTJmJ99nxgYpPueoaZ1GrFQtUDe9QnVLlHYTNDSDb90= +github.com/ipld/go-car/v2 v2.6.0/go.mod h1:qoqfgPnQYcaAYcfphctffdaNWJIWBR2QN4pjuKUtgao= github.com/ipld/go-codec-dagpb v1.5.0 h1:RspDRdsJpLfgCI0ONhTAnbHdySGD4t+LHSPK4X1+R0k= github.com/ipld/go-codec-dagpb v1.5.0/go.mod h1:0yRIutEFD8o1DGVqw4RSHh+BUTlJA9XWldxaaWR/o4g= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.19.0 h1:5axC7rJmPc17Emw6TelxGwnzALk0PdupZ2oj2roDj04= github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= +github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -432,7 +471,9 @@ github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlT github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -445,6 +486,7 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -455,6 +497,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= +github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -464,6 +507,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= @@ -482,6 +527,7 @@ github.com/libp2p/go-eventbus v0.2.1/go.mod h1:jc2S4SoEVPP48H9Wpzm5aiGwUCBMfGhVh github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= +github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= github.com/libp2p/go-libp2p v0.6.1/go.mod h1:CTFnWXogryAHjXAKEbOf1OWY+VeAP3lDMZkfEI5sT54= github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xSU1ivxn0k= github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw= @@ -606,6 +652,7 @@ github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbx github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= +github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= @@ -620,6 +667,7 @@ github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= +github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= @@ -642,10 +690,10 @@ github.com/libp2p/go-yamux v1.3.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZ github.com/libp2p/go-yamux v1.3.5/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= -github.com/libp2p/go-yamux v1.4.1 h1:P1Fe9vF4th5JOxxgQvfbOHkrGqIZniTLf+ddhZp8YTI= github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux/v2 v2.2.0/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ= github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rBfZQ= +github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= @@ -664,6 +712,7 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= @@ -680,6 +729,7 @@ github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= @@ -702,9 +752,11 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= @@ -795,15 +847,17 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.5.1 h1:auzK7OI497k6x4OvWq+TKAcpcSAlod0doAH72oIN0Jw= +github.com/onsi/ginkgo/v2 v2.5.1/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a h1:9iT75RHhYHWwWRlVWU7wnmtFulYcURCglzQOpT+cAF8= github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= @@ -822,10 +876,12 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -876,12 +932,19 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-18 v0.2.0 h1:5ViXqBZ90wpUcZS0ge79rf029yx0dYB0McyPJwqqj7U= +github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= github.com/quic-go/qtls-go1-19 v0.2.0 h1:Cvn2WdhyViFUHoOqK52i51k4nDX8EwIh5VJiVM4nttk= +github.com/quic-go/qtls-go1-19 v0.2.0/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= github.com/quic-go/qtls-go1-20 v0.1.0 h1:d1PK3ErFy9t7zxKsG3NXBJXZjp/kMLoIb3y/kV54oAI= +github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= github.com/quic-go/quic-go v0.32.0 h1:lY02md31s1JgPiiyfqJijpu/UX/Iun304FI3yUqX7tA= +github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo= github.com/quic-go/webtransport-go v0.5.1 h1:1eVb7WDWCRoaeTtFHpFBJ6WDN1bSrPrRoW6tZgSw0Ow= +github.com/quic-go/webtransport-go v0.5.1/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= +github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -924,6 +987,7 @@ github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5k github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -977,9 +1041,14 @@ github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= +github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= +github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= @@ -991,6 +1060,7 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= @@ -1033,7 +1103,9 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE= +go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= go.uber.org/fx v1.18.2 h1:bUNI6oShr+OVFQeU8cDNbnN7VFsu+SsjHzUF51V/GAU= +go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= @@ -1153,6 +1225,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1184,6 +1257,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1220,6 +1294,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1247,6 +1322,7 @@ golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.5.0 h1:+bSpV5HIeWkuvgaMfI3UmKRThoTA5ODJTUd8T17NO+4= @@ -1344,6 +1420,7 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= +nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/handlers.go b/handlers.go index 2b097c4..672d2c7 100644 --- a/handlers.go +++ b/handlers.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "github.com/ipfs/bifrost-gateway/lib" "math/rand" "net/http" "strconv" @@ -48,7 +49,7 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC } // Sets up a cache to store blocks in - cacheBlockStore, err := newCacheBlockStore(blockCacheSize) + cacheBlockStore, err := lib.NewCacheBlockStore(blockCacheSize) if err != nil { return nil, err } diff --git a/blockstore_cache.go b/lib/blockstore_cache.go similarity index 93% rename from blockstore_cache.go rename to lib/blockstore_cache.go index 800a363..66dc8dd 100644 --- a/blockstore_cache.go +++ b/lib/blockstore_cache.go @@ -1,23 +1,26 @@ -package main +package lib import ( "context" + "errors" "github.com/ipfs/go-cid" - format "github.com/ipfs/go-ipld-format" - "github.com/prometheus/client_golang/prometheus" - blockstore "github.com/ipfs/go-ipfs-blockstore" + format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-libipfs/blocks" + golog "github.com/ipfs/go-log/v2" lru "github.com/hashicorp/golang-lru/v2" + "github.com/prometheus/client_golang/prometheus" uatomic "go.uber.org/atomic" "go.uber.org/zap/zapcore" ) const DefaultCacheBlockStoreSize = 1024 -func newCacheBlockStore(size int) (blockstore.Blockstore, error) { +var goLog = golog.Logger("cache-blockstore") + +func NewCacheBlockStore(size int) (blockstore.Blockstore, error) { c, err := lru.New2Q[string, []byte](size) if err != nil { return nil, err @@ -126,7 +129,7 @@ func (l *cacheBlockStore) PutMany(ctx context.Context, blks []blocks.Block) erro } func (l *cacheBlockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { - return nil, errNotImplemented + return nil, errors.New("not implemented") } func (l *cacheBlockStore) HashOnRead(enabled bool) { diff --git a/main.go b/main.go index 11f6054..c66dc16 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( _ "embed" "errors" "fmt" + "github.com/ipfs/bifrost-gateway/lib" "log" "net/http" "os" @@ -54,7 +55,7 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, proxyGateway := getEnvs(EnvProxyGateway, "") kuboRPC := getEnvs(EnvKuboRPC, DefaultKuboPRC) - blockCacheSize, err := getEnvInt(EnvBlockCacheSize, DefaultCacheBlockStoreSize) + blockCacheSize, err := getEnvInt(EnvBlockCacheSize, lib.DefaultCacheBlockStoreSize) if err != nil { return err } From ff597a2648f03103d51fe1279e3ba69d813ed719 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 21 Mar 2023 17:35:13 -0400 Subject: [PATCH 03/27] start graph gateway impl --- go.mod | 55 ++++++++-- lib/graph_gateway.go | 238 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 lib/graph_gateway.go diff --git a/go.mod b/go.mod index 6e12fbe..0e2f1c8 100644 --- a/go.mod +++ b/go.mod @@ -10,12 +10,20 @@ require ( github.com/ipfs/go-cid v0.4.0 github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipfs-exchange-interface v0.2.0 + github.com/ipfs/go-ipfs-exchange-offline v0.3.0 github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-ipns v0.3.0 github.com/ipfs/go-libipfs v0.7.1-0.20230321154728-1c4716f9e834 github.com/ipfs/go-log/v2 v2.5.1 + github.com/ipfs/go-namesys v0.7.0 + github.com/ipfs/go-path v0.3.1 + github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa github.com/ipfs/interface-go-ipfs-core v0.11.1 + github.com/ipld/go-car v0.5.0 + github.com/ipld/go-car/v2 v2.6.0 github.com/libp2p/go-libp2p v0.25.1 + github.com/libp2p/go-libp2p-routing-helpers v0.4.0 + github.com/multiformats/go-multicodec v0.7.0 github.com/prometheus/client_golang v1.14.0 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 github.com/spf13/cobra v1.6.1 @@ -26,22 +34,36 @@ require ( require ( github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/containerd/cgroups v1.0.4 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect + github.com/cskr/pubsub v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/elastic/gosigar v0.14.2 // indirect + github.com/flynn/noise v1.0.0 // indirect + github.com/francoispqt/gojay v1.2.13 // indirect github.com/gabriel-vasile/mimetype v1.4.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gopacket v1.1.19 // indirect + github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/huin/goupnp v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/influxdata/tdigest v0.0.1 // indirect github.com/ipfs/bbloom v0.0.4 // indirect @@ -49,6 +71,7 @@ require ( github.com/ipfs/go-block-format v0.1.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-fetcher v1.6.1 // indirect + github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect @@ -57,61 +80,79 @@ require ( github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-merkledag v0.9.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect - github.com/ipfs/go-namesys v0.7.0 // indirect - github.com/ipfs/go-path v0.3.1 // indirect - github.com/ipfs/go-unixfs v0.4.4-0.20230301082657-5fd2773dcaaa // indirect github.com/ipfs/go-unixfsnode v1.5.2 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect - github.com/ipld/go-car v0.5.0 // indirect github.com/ipld/go-codec-dagpb v1.5.0 // indirect github.com/ipld/go-ipld-prime v0.19.0 // indirect + github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-doh-resolver v0.4.0 // indirect + github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect github.com/libp2p/go-libp2p-kad-dht v0.21.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect - github.com/libp2p/go-libp2p-routing-helpers v0.4.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect + github.com/libp2p/go-nat v0.1.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect + github.com/libp2p/go-reuseport v0.2.0 // indirect + github.com/libp2p/go-yamux/v4 v4.0.0 // indirect + github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.50 // indirect + github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect + github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr v0.8.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect + github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multicodec v0.7.0 // indirect github.com/multiformats/go-multihash v0.2.1 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect + github.com/onsi/ginkgo/v2 v2.5.1 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.39.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/quic-go/qpack v0.4.0 // indirect + github.com/quic-go/qtls-go1-18 v0.2.0 // indirect + github.com/quic-go/qtls-go1-19 v0.2.0 // indirect + github.com/quic-go/qtls-go1-20 v0.1.0 // indirect + github.com/quic-go/quic-go v0.32.0 // indirect + github.com/quic-go/webtransport-go v0.5.1 // indirect + github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect + github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.12.0 // indirect go.opentelemetry.io/otel/trace v1.12.0 // indirect + go.uber.org/dig v1.15.0 // indirect + go.uber.org/fx v1.18.2 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.5.0 // indirect golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 // indirect @@ -119,9 +160,11 @@ require ( golang.org/x/net v0.5.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect golang.org/x/tools v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect + nhooyr.io/websocket v1.8.7 // indirect ) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go new file mode 100644 index 0000000..fab6a83 --- /dev/null +++ b/lib/graph_gateway.go @@ -0,0 +1,238 @@ +package lib + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + + "github.com/filecoin-saturn/caboose" + "github.com/ipfs/go-blockservice" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/go-libipfs/gateway" + "github.com/ipfs/go-namesys" + "github.com/ipfs/go-namesys/resolve" + ipfspath "github.com/ipfs/go-path" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" + ifacepath "github.com/ipfs/interface-go-ipfs-core/path" + routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" + "github.com/multiformats/go-multicodec" +) + +// type DataCallback = func(resource string, reader io.Reader) error +// TODO: Don't use a caboose type, perhaps ask them to use a type alias instead of a type +type DataCallback = caboose.DataCallback + +type CarFetcher interface { + Fetch(ctx context.Context, path string, cb DataCallback) error +} + +type gwOptions struct { + ns namesys.NameSystem + vs routing.ValueStore + bs blockstore.Blockstore +} + +// WithNameSystem sets the name system to use for the gateway. If not set it will use a default DNSLink resolver +// along with any configured ValueStore +func WithNameSystem(ns namesys.NameSystem) GraphGatewayOption { + return func(opts *gwOptions) error { + opts.ns = ns + return nil + } +} + +// WithValueStore sets the ValueStore to use for the gateway +func WithValueStore(vs routing.ValueStore) GraphGatewayOption { + return func(opts *gwOptions) error { + opts.vs = vs + return nil + } +} + +// WithBlockstore sets the Blockstore to use for the gateway +func WithBlockstore(bs blockstore.Blockstore) GraphGatewayOption { + return func(opts *gwOptions) error { + opts.bs = bs + return nil + } +} + +type GraphGatewayOption func(gwOptions *gwOptions) error + +type GraphGateway struct { + fetcher CarFetcher + routing routing.ValueStore + namesys namesys.NameSystem + bstore blockstore.Blockstore + bsrv blockservice.BlockService +} + +func NewGraphGatewayBackend(f CarFetcher, opts ...GraphGatewayOption) (*GraphGateway, error) { + var compiledOptions gwOptions + for _, o := range opts { + if err := o(&compiledOptions); err != nil { + return nil, err + } + } + + // Setup a name system so that we are able to resolve /ipns links. + vs := compiledOptions.vs + if vs == nil { + vs = routinghelpers.Null{} + } + + ns := compiledOptions.ns + if ns == nil { + dns, err := gateway.NewDNSResolver(nil, nil) + if err != nil { + return nil, err + } + + ns, err = namesys.NewNameSystem(vs, namesys.WithDNSResolver(dns)) + if err != nil { + return nil, err + } + } + + bs := compiledOptions.bs + if compiledOptions.bs == nil { + // Sets up a cache to store blocks in + cbs, err := NewCacheBlockStore(1000) + if err != nil { + return nil, err + } + + // Set up support for identity hashes (https://github.com/ipfs/bifrost-gateway/issues/38) + cbs = blockstore.NewIdStore(cbs) + bs = cbs + } + + return &GraphGateway{ + fetcher: f, + routing: vs, + namesys: ns, + bstore: bs, + }, nil +} + +/* +Implementation iteration plan: + +1. Fetch CAR into per-request memory blockstore and serve response +2. Fetch CAR into shared memory blockstore and serve response along with a blockservice that does block requests for missing data +3. Start doing the walk locally and then if a path segment is incomplete send a request for blocks and upon every received block try to continue +4. Start doing the walk locally and keep a list of "plausible" blocks, if after issuing a request we get a non-plausible block then report them and attempt to recover by redoing the last segment +5. Don't redo the last segment fully if it's part of a UnixFS file and we can do range requests +*/ + +func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { + //TODO implement me + panic("implement me") +} + +func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePath, getRange ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) { + //TODO implement me + panic("implement me") +} + +func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { + //TODO implement me + panic("implement me") +} + +func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { + //TODO implement me + panic("implement me") +} + +func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { + //TODO implement me + panic("implement me") +} + +func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { + //TODO implement me + panic("implement me") +} + +func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) { + //TODO implement me + panic("implement me") +} + +func (api *GraphGateway) IsCached(ctx context.Context, path ifacepath.Path) bool { + //TODO implement me + panic("implement me") +} + +// TODO: This is copy-paste from blocks gateway, maybe share code +func (api *GraphGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + if api.routing == nil { + return nil, gateway.NewErrorResponse(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) + } + + // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful + // round trips to the remote routing provider. + if multicodec.Code(c.Type()) != multicodec.Libp2pKey { + return nil, gateway.NewErrorResponse(errors.New("cid codec must be libp2p-key"), http.StatusBadRequest) + } + + // The value store expects the key itself to be encoded as a multihash. + id, err := peer.FromCid(c) + if err != nil { + return nil, err + } + + return api.routing.GetValue(ctx, "/ipns/"+string(id)) +} + +// TODO: This is copy-paste from blocks gateway, maybe share code +func (api *GraphGateway) ResolveMutable(ctx context.Context, p ifacepath.Path) (gateway.ImmutablePath, error) { + err := p.IsValid() + if err != nil { + return gateway.ImmutablePath{}, err + } + + ipath := ipfspath.Path(p.String()) + switch ipath.Segments()[0] { + case "ipns": + ipath, err = resolve.ResolveIPNS(ctx, api.namesys, ipath) + if err != nil { + return gateway.ImmutablePath{}, err + } + imPath, err := gateway.NewImmutablePath(ifacepath.New(ipath.String())) + if err != nil { + return gateway.ImmutablePath{}, err + } + return imPath, nil + case "ipfs": + imPath, err := gateway.NewImmutablePath(ifacepath.New(ipath.String())) + if err != nil { + return gateway.ImmutablePath{}, err + } + return imPath, nil + default: + return gateway.ImmutablePath{}, gateway.NewErrorResponse(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) + } +} + +// TODO: This is copy-paste from blocks gateway, maybe share code +func (api *GraphGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { + if api.namesys != nil { + p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + if err == namesys.ErrResolveRecursion { + err = nil + } + return ifacepath.New(p.String()), err + } + + return nil, gateway.NewErrorResponse(errors.New("not implemented"), http.StatusNotImplemented) +} + +var _ gateway.IPFSBackend = (*GraphGateway)(nil) From ea196597146f6c958468d3f698ba7ad5c7ce5c2b Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 22 Mar 2023 00:58:05 -0400 Subject: [PATCH 04/27] trivial implementation --- go.mod | 3 ++ go.sum | 8 ++++ handlers.go | 51 ++++++++++++++---------- lib/graph_gateway.go | 94 ++++++++++++++++++++++++++++++++++++-------- main.go | 5 ++- 5 files changed, 122 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 0e2f1c8..8fa5f10 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-cid v0.4.0 + github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipfs-exchange-offline v0.3.0 @@ -57,6 +58,7 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect github.com/google/uuid v1.3.0 // indirect @@ -143,6 +145,7 @@ require ( github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect diff --git a/go.sum b/go.sum index 631626e..578da06 100644 --- a/go.sum +++ b/go.sum @@ -151,6 +151,7 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -232,6 +233,7 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -361,6 +363,8 @@ github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBR github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= +github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= +github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-fetcher v1.6.1 h1:UFuRVYX5AIllTiRhi5uK/iZkfhSpBCGX7L70nSZEmK8= github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo= github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= @@ -838,6 +842,7 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -847,6 +852,7 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.5.1 h1:auzK7OI497k6x4OvWq+TKAcpcSAlod0doAH72oIN0Jw= github.com/onsi/ginkgo/v2 v2.5.1/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -1035,6 +1041,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= @@ -1397,6 +1404,7 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= diff --git a/handlers.go b/handlers.go index 672d2c7..9e82978 100644 --- a/handlers.go +++ b/handlers.go @@ -41,32 +41,41 @@ func withRequestLogger(next http.Handler) http.Handler { }) } -func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockCacheSize int, cdns *cachedDNS) (*http.Server, error) { - // Sets up an exchange based on the given Block Store - exch, err := newExchange(bs) - if err != nil { - return nil, err - } +func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockCacheSize int, cdns *cachedDNS, useGraphGatewayBackend bool) (*http.Server, error) { + // Sets up the routing system, which will proxy the IPNS routing requests to the given gateway. + routing := newProxyRouting(kuboRPC, cdns) - // Sets up a cache to store blocks in - cacheBlockStore, err := lib.NewCacheBlockStore(blockCacheSize) - if err != nil { - return nil, err - } + var gwAPI gateway.IPFSBackend + var err error + if !useGraphGatewayBackend { + // Sets up an exchange based on the given Block Store + exch, err := newExchange(bs) + if err != nil { + return nil, err + } - // Set up support for identity hashes (https://github.com/ipfs/bifrost-gateway/issues/38) - cacheBlockStore = bstore.NewIdStore(cacheBlockStore) + // Sets up a cache to store blocks in + cacheBlockStore, err := lib.NewCacheBlockStore(blockCacheSize) + if err != nil { + return nil, err + } - // Sets up a blockservice which tries the cache and falls back to the exchange - blockService := blockservice.New(cacheBlockStore, exch) + // Set up support for identity hashes (https://github.com/ipfs/bifrost-gateway/issues/38) + cacheBlockStore = bstore.NewIdStore(cacheBlockStore) - // Sets up the routing system, which will proxy the IPNS routing requests to the given gateway. - routing := newProxyRouting(kuboRPC, cdns) + // Sets up a blockservice which tries the cache and falls back to the exchange + blockService := blockservice.New(cacheBlockStore, exch) - // Creates the gateway with the block service and the routing. - gwAPI, err := gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing)) - if err != nil { - return nil, err + // Creates the gateway with the block service and the routing. + gwAPI, err = gateway.NewBlocksGateway(blockService, gateway.WithValueStore(routing)) + if err != nil { + return nil, err + } + } else { + gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), lib.WithValueStore(routing)) + if err != nil { + return nil, err + } } headers := map[string][]string{} diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index fab6a83..4638249 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + leveldb "github.com/ipfs/go-ds-leveldb" + offline "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipld/go-car" "io" "net/http" @@ -131,44 +134,103 @@ Implementation iteration plan: 5. Don't redo the last segment fully if it's part of a UnixFS file and we can do range requests */ +func (api *GraphGateway) loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx context.Context, path string) (blockstore.Blockstore, gateway.IPFSBackend, error) { + ds, err := leveldb.NewDatastore("", nil) + if err != nil { + return nil, nil, err + } + bstore := blockstore.NewBlockstore(ds) + + err = api.fetcher.Fetch(ctx, path, func(resource string, reader io.Reader) error { + cr, err := car.NewCarReader(reader) + if err != nil { + return err + } + for { + blk, err := cr.Next() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + if err := bstore.Put(ctx, blk); err != nil { + return err + } + } + }) + if err != nil { + return nil, nil, err + } + + bserv := blockservice.New(bstore, offline.Exchange(bstore)) + blkgw, err := gateway.NewBlocksGateway(bserv) + if err != nil { + return nil, nil, err + } + + return bstore, blkgw, nil +} + func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { - //TODO implement me - panic("implement me") + _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + return blkgw.Get(ctx, path) } func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePath, getRange ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) { - //TODO implement me - panic("implement me") + // TODO: actually implement ranges + _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + return blkgw.GetRange(ctx, path) } func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - //TODO implement me - panic("implement me") + _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=all") + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + return blkgw.GetAll(ctx, path) } func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { - //TODO implement me - panic("implement me") + _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + return blkgw.GetBlock(ctx, path) } func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - //TODO implement me - panic("implement me") + _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&bytes=0:1023") + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + return blkgw.Head(ctx, path) } func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { - //TODO implement me - panic("implement me") + _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") + if err != nil { + return gateway.ContentPathMetadata{}, err + } + return blkgw.ResolvePath(ctx, path) } func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) { - //TODO implement me - panic("implement me") + _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car") + if err != nil { + return gateway.ContentPathMetadata{}, nil, nil, err + } + return blkgw.GetCAR(ctx, path) } func (api *GraphGateway) IsCached(ctx context.Context, path ifacepath.Path) bool { - //TODO implement me - panic("implement me") + return false } // TODO: This is copy-paste from blocks gateway, maybe share code diff --git a/main.go b/main.go index c66dc16..cfc49bc 100644 --- a/main.go +++ b/main.go @@ -35,7 +35,7 @@ const ( func init() { rootCmd.Flags().Int("gateway-port", 8081, "gateway port") rootCmd.Flags().Int("metrics-port", 8041, "metrics port") - + rootCmd.Flags().Bool("graph-gateway", false, "use a graph fetching based gateway") } var rootCmd = &cobra.Command{ @@ -49,6 +49,7 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, // Get flags. gatewayPort, _ := cmd.Flags().GetInt("gateway-port") metricsPort, _ := cmd.Flags().GetInt("metrics-port") + useGraphGateway, _ := cmd.Flags().GetBool("graph-gateway") // Get env variables. saturnOrchestrator := getEnv(EnvSaturnOrchestrator, "") @@ -87,7 +88,7 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, log.Fatalf("Unable to start. bifrost-gateway requires either PROXY_GATEWAY_URL or STRN_ORCHESTRATOR_URL to be set.\n\nRead docs at https://github.com/ipfs/bifrost-gateway/blob/main/docs/environment-variables.md\n\n") } - gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, gatewayPort, blockCacheSize, cdns) + gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, gatewayPort, blockCacheSize, cdns, useGraphGateway) if err != nil { return err } From 386e3c14272eae053db80fe3de8962cf6d903536 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 22 Mar 2023 14:39:24 -0400 Subject: [PATCH 05/27] add async --- lib/graph_gateway.go | 124 ++++++++++++++++++++++++++----------- lib/pubsub.go | 143 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+), 35 deletions(-) create mode 100644 lib/pubsub.go diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 4638249..3f344b9 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -4,16 +4,13 @@ import ( "context" "errors" "fmt" - leveldb "github.com/ipfs/go-ds-leveldb" - offline "github.com/ipfs/go-ipfs-exchange-offline" - "github.com/ipld/go-car" - "io" - "net/http" - "github.com/filecoin-saturn/caboose" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" + leveldb "github.com/ipfs/go-ds-leveldb" blockstore "github.com/ipfs/go-ipfs-blockstore" + exchange "github.com/ipfs/go-ipfs-exchange-interface" + "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-libipfs/files" "github.com/ipfs/go-libipfs/gateway" "github.com/ipfs/go-namesys" @@ -21,10 +18,14 @@ import ( ipfspath "github.com/ipfs/go-path" nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" ifacepath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipld/go-car" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" + "io" + "net/http" ) // type DataCallback = func(resource string, reader io.Reader) error @@ -134,46 +135,56 @@ Implementation iteration plan: 5. Don't redo the last segment fully if it's part of a UnixFS file and we can do range requests */ -func (api *GraphGateway) loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx context.Context, path string) (blockstore.Blockstore, gateway.IPFSBackend, error) { +func (api *GraphGateway) loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx context.Context, path string) (gateway.IPFSBackend, error) { ds, err := leveldb.NewDatastore("", nil) if err != nil { - return nil, nil, err + return nil, err } bstore := blockstore.NewBlockstore(ds) + exch := newInboundBlockExchange() - err = api.fetcher.Fetch(ctx, path, func(resource string, reader io.Reader) error { - cr, err := car.NewCarReader(reader) - if err != nil { - return err - } - for { - blk, err := cr.Next() + doneWithFetcher := make(chan error, 1) + + go func() { + defer func() { + if r := recover(); r != nil { + fmt.Println("Recovered fetcher error", r) + } + }() + doneWithFetcher <- api.fetcher.Fetch(ctx, path, func(resource string, reader io.Reader) error { + cr, err := car.NewCarReader(reader) if err != nil { - if errors.Is(err, io.EOF) { - return nil - } return err } - if err := bstore.Put(ctx, blk); err != nil { - return err + for { + blk, err := cr.Next() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + if err := bstore.Put(ctx, blk); err != nil { + return err + } + if err := exch.NotifyNewBlocks(ctx, blk); err != nil { + return err + } } - } - }) - if err != nil { - return nil, nil, err - } + }) + }() - bserv := blockservice.New(bstore, offline.Exchange(bstore)) + bserv := blockservice.New(bstore, exch) blkgw, err := gateway.NewBlocksGateway(bserv) if err != nil { - return nil, nil, err + return nil, err } - return bstore, blkgw, nil + return blkgw, nil } func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { - _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") + blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -182,7 +193,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath) (g func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePath, getRange ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) { // TODO: actually implement ranges - _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") + blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -190,7 +201,7 @@ func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePat } func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=all") + blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=all") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -198,7 +209,7 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) } func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { - _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") + blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -206,7 +217,7 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat } func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&bytes=0:1023") + blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&bytes=0:1023") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -214,7 +225,7 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( } func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { - _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") + blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") if err != nil { return gateway.ContentPathMetadata{}, err } @@ -222,7 +233,7 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable } func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) { - _, blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car") + blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car") if err != nil { return gateway.ContentPathMetadata{}, nil, nil, err } @@ -298,3 +309,46 @@ func (api *GraphGateway) GetDNSLinkRecord(ctx context.Context, hostname string) } var _ gateway.IPFSBackend = (*GraphGateway)(nil) + +type inboundBlockExchange struct { + ps PubSub +} + +func newInboundBlockExchange() *inboundBlockExchange { + return &inboundBlockExchange{ + ps: NewPubSub(), + } +} + +func (i *inboundBlockExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + blk := <-i.ps.Subscribe(ctx, c.Hash()) + if err := ctx.Err(); err != nil { + return nil, err + } + return blk, nil +} + +func (i *inboundBlockExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { + mhMap := make(map[string]struct{}) + for _, c := range cids { + mhMap[string(c.Hash())] = struct{}{} + } + mhs := make([]multihash.Multihash, 0, len(mhMap)) + for k := range mhMap { + mhs = append(mhs, multihash.Multihash(k)) + } + return i.ps.Subscribe(ctx, mhs...), nil +} + +func (i *inboundBlockExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { + // TODO: handle context cancellation and/or blockage here + i.ps.Publish(blocks...) + return nil +} + +func (i *inboundBlockExchange) Close() error { + i.ps.Shutdown() + return nil +} + +var _ exchange.Interface = (*inboundBlockExchange)(nil) diff --git a/lib/pubsub.go b/lib/pubsub.go new file mode 100644 index 0000000..07d8299 --- /dev/null +++ b/lib/pubsub.go @@ -0,0 +1,143 @@ +package lib + +import ( + "context" + "github.com/multiformats/go-multihash" + "sync" + + pubsub "github.com/cskr/pubsub" + blocks "github.com/ipfs/go-libipfs/blocks" +) + +// Note: blantantly copied from boxo/bitswap internals https://github.com/ipfs/boxo/blob/664b3e5ee4a997c67909da9b017a28efa40cc8ae/bitswap/client/internal/notifications/notifications.go +// some minor changes were made + +const bufferSize = 16 + +// PubSub is a simple interface for publishing blocks and being able to subscribe +// for multihashes. It's used internally by an exchange to decouple receiving blocks +// and actually providing them back to the GetBlocks caller. +// Note: because multihashes are being requested and blocks returned the codecs could be anything +type PubSub interface { + Publish(blocks ...blocks.Block) + Subscribe(ctx context.Context, keys ...multihash.Multihash) <-chan blocks.Block + Shutdown() +} + +// NewPubSub generates a new PubSub interface. +func NewPubSub() PubSub { + return &impl{ + wrapped: *pubsub.New(bufferSize), + closed: make(chan struct{}), + } +} + +type impl struct { + lk sync.RWMutex + wrapped pubsub.PubSub + + closed chan struct{} +} + +func (ps *impl) Publish(blocks ...blocks.Block) { + ps.lk.RLock() + defer ps.lk.RUnlock() + select { + case <-ps.closed: + return + default: + } + + for _, block := range blocks { + ps.wrapped.Pub(block, string(block.Cid().Hash())) + } +} + +func (ps *impl) Shutdown() { + ps.lk.Lock() + defer ps.lk.Unlock() + select { + case <-ps.closed: + return + default: + } + close(ps.closed) + ps.wrapped.Shutdown() +} + +// Subscribe returns a channel of blocks for the given |keys|. |blockChannel| +// is closed if the |ctx| times out or is cancelled, or after receiving the blocks +// corresponding to |keys|. +func (ps *impl) Subscribe(ctx context.Context, keys ...multihash.Multihash) <-chan blocks.Block { + + blocksCh := make(chan blocks.Block, len(keys)) + valuesCh := make(chan interface{}, len(keys)) // provide our own channel to control buffer, prevent blocking + if len(keys) == 0 { + close(blocksCh) + return blocksCh + } + + // prevent shutdown + ps.lk.RLock() + defer ps.lk.RUnlock() + + select { + case <-ps.closed: + close(blocksCh) + return blocksCh + default: + } + + // AddSubOnceEach listens for each key in the list, and closes the channel + // once all keys have been received + ps.wrapped.AddSubOnceEach(valuesCh, toStrings(keys)...) + go func() { + defer func() { + close(blocksCh) + + ps.lk.RLock() + defer ps.lk.RUnlock() + // Don't touch the pubsub instance if we're + // already closed. + select { + case <-ps.closed: + return + default: + } + + ps.wrapped.Unsub(valuesCh) + }() + + for { + select { + case <-ctx.Done(): + return + case <-ps.closed: + case val, ok := <-valuesCh: + if !ok { + return + } + block, ok := val.(blocks.Block) + if !ok { + return + } + select { + case <-ctx.Done(): + return + case blocksCh <- block: // continue + case <-ps.closed: + } + } + } + }() + + return blocksCh +} + +func toStrings(keys []multihash.Multihash) []string { + strs := make([]string, 0, len(keys)) + for _, key := range keys { + strs = append(strs, string(key)) + } + return strs +} From 34e97fbff5630ca63a716b11af13f236ae6b7127 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 23 Mar 2023 11:29:48 -0400 Subject: [PATCH 06/27] update testing backend --- backend/handlers.go | 249 ++++++++++++++++++++++++++++++++++++++++++-- backend/main.go | 17 ++- 2 files changed, 253 insertions(+), 13 deletions(-) diff --git a/backend/handlers.go b/backend/handlers.go index 68855f6..2d0bb36 100644 --- a/backend/handlers.go +++ b/backend/handlers.go @@ -4,15 +4,29 @@ import ( "bytes" "context" "fmt" + bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice" + format "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/go-libipfs/gateway" + "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-path" + "github.com/ipfs/go-path/resolver" unixfile "github.com/ipfs/go-unixfs/file" + "github.com/ipfs/go-unixfsnode" "github.com/ipld/go-car" "github.com/ipld/go-car/util" + dagpb "github.com/ipld/go-codec-dagpb" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" "io" "net/http" - "os" + "net/url" "runtime/debug" "strconv" "strings" + "sync" "time" "github.com/prometheus/client_golang/prometheus" @@ -76,7 +90,7 @@ func makeGatewayCARHandler(bsrv blockservice.BlockService, port int) (*http.Serv if formatParam := r.URL.Query().Get("format"); formatParam != "" { isCar = formatParam == "car" if !isCar { - http.Error(w, "only raw format supported", http.StatusBadRequest) + http.Error(w, "only car format supported", http.StatusBadRequest) return } } else { @@ -104,7 +118,7 @@ func makeGatewayCARHandler(bsrv blockservice.BlockService, port int) (*http.Serv return } - carStream, err := simpleSelectorToCar(contentPath) + carStream, err := simpleSelectorToCar(ctx, bsrv, contentPath.String(), r.URL.Query()) if err != nil { http.Error(w, "only the ipfs names is supported", http.StatusBadRequest) return @@ -147,8 +161,8 @@ func makeGatewayCARHandler(bsrv blockservice.BlockService, port int) (*http.Serv }, nil } -func simpleSelectorToCar(ipfsPath ipath.Path) (io.ReadCloser, error) { - pathSegs := strings.Split(ipfsPath.String(), "/") +func simpleSelectorToCar(ctx context.Context, bsrv blockservice.BlockService, p string, params url.Values) (io.ReadCloser, error) { + pathSegs := strings.Split(p, "/") if len(pathSegs) < 3 || !(pathSegs[0] == "" && pathSegs[1] == "ipfs") { return nil, fmt.Errorf("invalid path") } @@ -159,6 +173,11 @@ func simpleSelectorToCar(ipfsPath ipath.Path) (io.ReadCloser, error) { return nil, err } + ipfspath, err := path.ParsePath(p) + if err != nil { + return nil, err + } + r, w := io.Pipe() // Setup header for the output car err = car.WriteHeader(&car.CarHeader{ @@ -169,20 +188,230 @@ func simpleSelectorToCar(ipfsPath ipath.Path) (io.ReadCloser, error) { return nil, fmt.Errorf("writing car header: %w", err) } + rangeStr, hasRange := params.Get("bytes"), params.Has("bytes") + depthStr, hasDepth := params.Get("depth"), params.Has("depth") + + if hasDepth && !(depthStr == "0" || depthStr == "1" || depthStr == "all") { + return nil, fmt.Errorf("depth type: %s not supported", depthStr) + } + var getRange *gateway.GetRange + if hasRange { + getRange, err = rangeStrToGetRange(rangeStr) + if err != nil { + return nil, err + } + } + go func() { defer w.Close() - remainingPath := pathSegs[1:] - unixfile.NewUnixfsFile() + blockGetter := merkledag.NewDAGService(bsrv).Session(ctx) + blockGetter = &nodeGetterToCarExporer{ + ng: blockGetter, + w: w, + mhSet: make(map[string]struct{}), + } + dsrv := merkledag.NewReadOnlyDagService(blockGetter) + + // Setup the UnixFS resolver. + fetcherConfig := bsfetcher.NewFetcherConfig(bsrv) + fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) + r := resolver.NewBasicResolver(fetcher) + + lastCid, remainder, err := r.ResolveToLastNode(ctx, ipfspath) + if err != nil { + goLog.Error(err) + return + } + + if hasDepth && depthStr == "0" { + return + } + + lastCidNode, err := dsrv.Get(ctx, lastCid) + if err != nil { + goLog.Error(err) + return + } - err = util.LdWrite(os.Stdout, block.Cid().Bytes(), block.RawData()) // write to the output car + ufsNode, err := unixfile.NewUnixfsFile(ctx, dsrv, lastCidNode) if err != nil { - return fmt.Errorf("writing to output car: %w", err) + // It's not UnixFS + + // If it's all fetch the graph recursively + if depthStr == "all" { + if err := merkledag.FetchGraph(ctx, lastCid, dsrv); err != nil { + goLog.Error(err) + } + return + } + + //if not then either this is an error (which we can't report) or this is the last block for us to return + return + } + if f, ok := ufsNode.(files.File); ok { + if len(remainder) > 0 { + // this is an error, so we're done + return + } + if hasRange { + // TODO: testing + check off by one errors + var numToRead int64 + if *getRange.To < 0 { + size, err := f.Seek(0, io.SeekEnd) + if err != nil { + return + } + numToRead = (size - *getRange.To) - int64(getRange.From) + } else { + numToRead = int64(getRange.From) - *getRange.To + } + + if _, err := f.Seek(int64(getRange.From), io.SeekStart); err != nil { + return + } + _, _ = io.CopyN(io.Discard, f, numToRead) + return + } + } else if d, ok := ufsNode.(files.Directory); ok { + if depthStr == "1" { + for d.Entries().Next() { + } + return + } + if depthStr == "all" { + // TODO: being lazy here + w, err := files.NewTarWriter(io.Discard) + if err != nil { + goLog.Error(fmt.Errorf("could not create tar write %w", err)) + return + } + if err := w.WriteFile(d, "tmp"); err != nil { + goLog.Error(err) + return + } + return + } + } else { + return } }() - _ = rootCid return r, nil } +type nodeGetterToCarExporer struct { + ng format.NodeGetter + w io.Writer + + lk sync.RWMutex + mhSet map[string]struct{} +} + +func (n *nodeGetterToCarExporer) Get(ctx context.Context, c cid.Cid) (format.Node, error) { + nd, err := n.ng.Get(ctx, c) + if err != nil { + return nil, err + } + + if err := n.trySendBlock(nd); err != nil { + return nil, err + } + + return nd, nil +} + +func (n *nodeGetterToCarExporer) GetMany(ctx context.Context, cids []cid.Cid) <-chan *format.NodeOption { + ndCh := n.ng.GetMany(ctx, cids) + outCh := make(chan *format.NodeOption) + go func() { + defer close(outCh) + for nd := range ndCh { + if nd.Err == nil { + if err := n.trySendBlock(nd.Node); err != nil { + select { + case outCh <- &format.NodeOption{Err: err}: + case <-ctx.Done(): + } + return + } + select { + case outCh <- nd: + case <-ctx.Done(): + } + } + } + }() + return outCh +} + +func (n *nodeGetterToCarExporer) trySendBlock(block blocks.Block) error { + h := string(block.Cid().Hash()) + n.lk.RLock() + _, found := n.mhSet[h] + n.lk.RUnlock() + if !found { + doSend := false + n.lk.Lock() + _, found := n.mhSet[h] + if !found { + doSend = true + n.mhSet[h] = struct{}{} + } + n.lk.Unlock() + if doSend { + err := util.LdWrite(n.w, block.Cid().Bytes(), block.RawData()) // write to the output car + if err != nil { + return fmt.Errorf("writing to output car: %w", err) + } + } + } + return nil +} + +var _ format.NodeGetter = (*nodeGetterToCarExporer)(nil) + +func rangeStrToGetRange(rangeStr string) (*gateway.GetRange, error) { + rangeElems := strings.Split(rangeStr, ":") + if len(rangeElems) > 2 { + return nil, fmt.Errorf("invalid range") + } + first, err := strconv.ParseUint(rangeElems[0], 10, 64) + if err != nil { + return nil, err + } + + if rangeElems[1] == "*" { + return &gateway.GetRange{ + From: first, + To: nil, + }, nil + } + + second, err := strconv.ParseInt(rangeElems[1], 10, 64) + if err != nil { + return nil, err + } + + if second < 0 { + // TODO: fix, might also require a fix in boxo/gateway + return nil, fmt.Errorf("unsupported") + } + + if uint64(second) < first { + return nil, fmt.Errorf("invalid range") + } + + return &gateway.GetRange{ + From: first, + To: &second, + }, nil +} + func makeGatewayBlockHandler(bsrv blockservice.BlockService, port int) (*http.Server, error) { mux := http.NewServeMux() mux.HandleFunc("/ipfs/", func(w http.ResponseWriter, r *http.Request) { diff --git a/backend/main.go b/backend/main.go index 35276a8..c159129 100644 --- a/backend/main.go +++ b/backend/main.go @@ -34,6 +34,7 @@ func init() { rootCmd.Flags().Int("gateway-port", 8081, "gateway port") rootCmd.Flags().Int("metrics-port", 8041, "metrics port") rootCmd.Flags().String("car-blockstore", "", "a CAR file to use for serving data instead of network requests") + golog.SetLogLevel("bifrost-gateway-backend", "debug") } var rootCmd = &cobra.Command{ @@ -80,9 +81,19 @@ var rootCmd = &cobra.Command{ log.Printf("Starting %s %s", name, version) - gatewaySrv, err := makeGatewayBlockHandler(bsrv, gatewayPort) - if err != nil { - return err + var gatewaySrv *http.Server + var err error + + if true { + gatewaySrv, err = makeGatewayCARHandler(bsrv, gatewayPort) + if err != nil { + return err + } + } else { + gatewaySrv, err = makeGatewayBlockHandler(bsrv, gatewayPort) + if err != nil { + return err + } } metricsSrv, err := makeMetricsHandler(metricsPort) From d496baebfd22154c8c249487d4615b42ebe40856 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 23 Mar 2023 15:22:16 -0400 Subject: [PATCH 07/27] backend fixes --- backend/handlers.go | 169 +++++++++++++++++++++++++++++++++++--------- backend/main.go | 13 +++- blockstore_proxy.go | 31 ++++++++ 3 files changed, 177 insertions(+), 36 deletions(-) diff --git a/backend/handlers.go b/backend/handlers.go index 2d0bb36..c4def40 100644 --- a/backend/handlers.go +++ b/backend/handlers.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "fmt" - bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice" + "github.com/ipfs/go-fetcher" format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-libipfs/files" @@ -18,8 +18,11 @@ import ( "github.com/ipld/go-car/util" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/schema" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" "io" "net/http" "net/url" @@ -179,14 +182,6 @@ func simpleSelectorToCar(ctx context.Context, bsrv blockservice.BlockService, p } r, w := io.Pipe() - // Setup header for the output car - err = car.WriteHeader(&car.CarHeader{ - Roots: []cid.Cid{rootCid}, - Version: 1, - }, w) - if err != nil { - return nil, fmt.Errorf("writing car header: %w", err) - } rangeStr, hasRange := params.Get("bytes"), params.Has("bytes") depthStr, hasDepth := params.Get("depth"), params.Has("depth") @@ -204,6 +199,16 @@ func simpleSelectorToCar(ctx context.Context, bsrv blockservice.BlockService, p go func() { defer w.Close() + + // Setup header for the output car + err = car.WriteHeader(&car.CarHeader{ + Roots: []cid.Cid{rootCid}, + Version: 1, + }, w) + if err != nil { + goLog.Error(fmt.Errorf("writing car header: %w", err)) + } + blockGetter := merkledag.NewDAGService(bsrv).Session(ctx) blockGetter = &nodeGetterToCarExporer{ ng: blockGetter, @@ -213,17 +218,10 @@ func simpleSelectorToCar(ctx context.Context, bsrv blockservice.BlockService, p dsrv := merkledag.NewReadOnlyDagService(blockGetter) // Setup the UnixFS resolver. - fetcherConfig := bsfetcher.NewFetcherConfig(bsrv) - fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) - r := resolver.NewBasicResolver(fetcher) + f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter) + pathResolver := resolver.NewBasicResolver(f) - lastCid, remainder, err := r.ResolveToLastNode(ctx, ipfspath) + lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, ipfspath) if err != nil { goLog.Error(err) return @@ -259,28 +257,37 @@ func simpleSelectorToCar(ctx context.Context, bsrv blockservice.BlockService, p // this is an error, so we're done return } - if hasRange { - // TODO: testing + check off by one errors - var numToRead int64 - if *getRange.To < 0 { - size, err := f.Seek(0, io.SeekEnd) - if err != nil { - return - } - numToRead = (size - *getRange.To) - int64(getRange.From) - } else { - numToRead = int64(getRange.From) - *getRange.To + + if !hasRange { + nw, err := io.Copy(io.Discard, f) + goLog.Debugf("nwritten %d", nw) + if err != nil { + goLog.Error(err) } + return + } - if _, err := f.Seek(int64(getRange.From), io.SeekStart); err != nil { + // TODO: testing + check off by one errors + var numToRead int64 + if *getRange.To < 0 { + size, err := f.Seek(0, io.SeekEnd) + if err != nil { return } - _, _ = io.CopyN(io.Discard, f, numToRead) + numToRead = (size - *getRange.To) - int64(getRange.From) + } else { + numToRead = int64(getRange.From) - *getRange.To + } + + if _, err := f.Seek(int64(getRange.From), io.SeekStart); err != nil { return } + _, _ = io.CopyN(io.Discard, f, numToRead) + return } else if d, ok := ufsNode.(files.Directory); ok { if depthStr == "1" { - for d.Entries().Next() { + iter := d.Entries() + for iter.Next() { } return } @@ -375,6 +382,102 @@ func (n *nodeGetterToCarExporer) trySendBlock(block blocks.Block) error { var _ format.NodeGetter = (*nodeGetterToCarExporer)(nil) +type nodeGetterFetcherSingleUseFactory struct { + linkSystem ipld.LinkSystem + protoChooser traversal.LinkTargetNodePrototypeChooser +} + +func newNodeGetterFetcherSingleUseFactory(ctx context.Context, ng format.NodeGetter) *nodeGetterFetcherSingleUseFactory { + ls := cidlink.DefaultLinkSystem() + ls.TrustedStorage = true + ls.StorageReadOpener = blockOpener(ctx, ng) + ls.NodeReifier = unixfsnode.Reify + + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + + return &nodeGetterFetcherSingleUseFactory{ls, pc} +} + +func (n *nodeGetterFetcherSingleUseFactory) NewSession(ctx context.Context) fetcher.Fetcher { + return n +} + +func (n *nodeGetterFetcherSingleUseFactory) NodeMatching(ctx context.Context, root ipld.Node, selector ipld.Node, cb fetcher.FetchCallback) error { + return n.nodeMatching(ctx, n.blankProgress(ctx), root, selector, cb) +} + +func (n *nodeGetterFetcherSingleUseFactory) BlockOfType(ctx context.Context, link ipld.Link, nodePrototype ipld.NodePrototype) (ipld.Node, error) { + return n.linkSystem.Load(ipld.LinkContext{}, link, nodePrototype) +} + +func (n *nodeGetterFetcherSingleUseFactory) BlockMatchingOfType(ctx context.Context, root ipld.Link, selector ipld.Node, nodePrototype ipld.NodePrototype, cb fetcher.FetchCallback) error { + // retrieve first node + prototype, err := n.PrototypeFromLink(root) + if err != nil { + return err + } + node, err := n.BlockOfType(ctx, root, prototype) + if err != nil { + return err + } + + progress := n.blankProgress(ctx) + progress.LastBlock.Link = root + return n.nodeMatching(ctx, progress, node, selector, cb) +} + +func (n *nodeGetterFetcherSingleUseFactory) PrototypeFromLink(lnk ipld.Link) (ipld.NodePrototype, error) { + return n.protoChooser(lnk, ipld.LinkContext{}) +} + +func (n *nodeGetterFetcherSingleUseFactory) nodeMatching(ctx context.Context, initialProgress traversal.Progress, node ipld.Node, match ipld.Node, cb fetcher.FetchCallback) error { + matchSelector, err := selector.ParseSelector(match) + if err != nil { + return err + } + return initialProgress.WalkMatching(node, matchSelector, func(prog traversal.Progress, n ipld.Node) error { + return cb(fetcher.FetchResult{ + Node: n, + Path: prog.Path, + LastBlockPath: prog.LastBlock.Path, + LastBlockLink: prog.LastBlock.Link, + }) + }) +} + +func (n *nodeGetterFetcherSingleUseFactory) blankProgress(ctx context.Context) traversal.Progress { + return traversal.Progress{ + Cfg: &traversal.Config{ + LinkSystem: n.linkSystem, + LinkTargetNodePrototypeChooser: n.protoChooser, + }, + } +} + +func blockOpener(ctx context.Context, ng format.NodeGetter) ipld.BlockReadOpener { + return func(_ ipld.LinkContext, lnk ipld.Link) (io.Reader, error) { + cidLink, ok := lnk.(cidlink.Link) + if !ok { + return nil, fmt.Errorf("invalid link type for loading: %v", lnk) + } + + blk, err := ng.Get(ctx, cidLink.Cid) + if err != nil { + return nil, err + } + + return bytes.NewReader(blk.RawData()), nil + } +} + +var _ fetcher.Fetcher = (*nodeGetterFetcherSingleUseFactory)(nil) +var _ fetcher.Factory = (*nodeGetterFetcherSingleUseFactory)(nil) + func rangeStrToGetRange(rangeStr string) (*gateway.GetRange, error) { rangeElems := strings.Split(rangeStr, ":") if len(rangeElems) > 2 { diff --git a/backend/main.go b/backend/main.go index c159129..3d27bef 100644 --- a/backend/main.go +++ b/backend/main.go @@ -11,6 +11,9 @@ import ( golog "github.com/ipfs/go-log/v2" carbs "github.com/ipld/go-car/v2/blockstore" "github.com/libp2p/go-libp2p" + dht "github.com/libp2p/go-libp2p-kad-dht" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/routing" "github.com/spf13/cobra" "log" "net/http" @@ -49,7 +52,7 @@ var rootCmd = &cobra.Command{ carbsLocation, _ := cmd.Flags().GetString("car-blockstore") var bsrv blockservice.BlockService - if carbsLocation == "" { + if carbsLocation != "" { bs, err := carbs.OpenReadOnly(carbsLocation) if err != nil { return err @@ -67,11 +70,15 @@ var rootCmd = &cobra.Command{ return err } - h, err := libp2p.New() + var r routing.Routing + h, err := libp2p.New(libp2p.Routing(func(host host.Host) (routing.PeerRouting, error) { + r, err = dht.New(cmd.Context(), host, dht.BootstrapPeersFunc(dht.GetDefaultBootstrapPeerAddrInfos)) + return r, err + })) if err != nil { return err } - n := network.NewFromIpfsHost(h, nil) + n := network.NewFromIpfsHost(h, r) bsc := client.New(cmd.Context(), n, bs) n.Start(bsc) defer n.Stop() diff --git a/blockstore_proxy.go b/blockstore_proxy.go index fc71c35..0c12c3e 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "github.com/ipfs/bifrost-gateway/lib" "io" "log" "math/rand" @@ -35,6 +36,36 @@ type proxyBlockStore struct { rand *rand.Rand } +func (ps *proxyBlockStore) Fetch(ctx context.Context, path string, cb lib.DataCallback) error { + u, err := url.Parse(fmt.Sprintf("%s/%s", ps.getRandomGatewayURL(), path)) + if err != nil { + return err + } + resp, err := ps.httpClient.Do(&http.Request{ + Method: http.MethodGet, + URL: u, + Header: http.Header{ + "Accept": []string{"application/vnd.ipld.car"}, + }, + }) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("http error from car gateway: %s", resp.Status) + } + + err = cb(path, resp.Body) + if err != nil { + resp.Body.Close() + return err + } + return resp.Body.Close() +} + +var _ lib.CarFetcher = (*proxyBlockStore)(nil) + func newProxyBlockStore(gatewayURL []string, cdns *cachedDNS) blockstore.Blockstore { s := rand.NewSource(time.Now().Unix()) rand := rand.New(s) From ce1ffd4adef5e2580eddbc18d9e98f9205e05f27 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 23 Mar 2023 15:23:15 -0400 Subject: [PATCH 08/27] change backend default ports --- backend/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/main.go b/backend/main.go index 3d27bef..2bc18bc 100644 --- a/backend/main.go +++ b/backend/main.go @@ -34,8 +34,8 @@ func main() { } func init() { - rootCmd.Flags().Int("gateway-port", 8081, "gateway port") - rootCmd.Flags().Int("metrics-port", 8041, "metrics port") + rootCmd.Flags().Int("gateway-port", 8082, "gateway port") + rootCmd.Flags().Int("metrics-port", 8042, "metrics port") rootCmd.Flags().String("car-blockstore", "", "a CAR file to use for serving data instead of network requests") golog.SetLogLevel("bifrost-gateway-backend", "debug") } From d8b4d4e577f9e5e2960a0b5d8da9543913c18962 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 24 Mar 2023 07:13:02 -0400 Subject: [PATCH 09/27] feat(graph-gateway): add some (hacky) support for fetching blocks as well as CARs --- blockstore_proxy.go | 2 +- handlers.go | 8 ++- lib/graph_gateway.go | 158 +++++++++++++++++++++++++++++++++++++++---- 3 files changed, 151 insertions(+), 17 deletions(-) diff --git a/blockstore_proxy.go b/blockstore_proxy.go index 0c12c3e..5f17450 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -3,7 +3,6 @@ package main import ( "context" "fmt" - "github.com/ipfs/bifrost-gateway/lib" "io" "log" "math/rand" @@ -11,6 +10,7 @@ import ( "net/url" "time" + "github.com/ipfs/bifrost-gateway/lib" "github.com/ipfs/go-cid" blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-libipfs/blocks" diff --git a/handlers.go b/handlers.go index 9e82978..23136b1 100644 --- a/handlers.go +++ b/handlers.go @@ -72,7 +72,13 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC return nil, err } } else { - gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), lib.WithValueStore(routing)) + // Sets up an exchange based on the given Block Store + exch, err := newExchange(bs) + if err != nil { + return nil, err + } + + gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), exch, lib.WithValueStore(routing)) if err != nil { return nil, err } diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 3f344b9..5edda8b 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -24,6 +24,7 @@ import ( "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" + "go.uber.org/multierr" "io" "net/http" ) @@ -70,14 +71,15 @@ func WithBlockstore(bs blockstore.Blockstore) GraphGatewayOption { type GraphGatewayOption func(gwOptions *gwOptions) error type GraphGateway struct { - fetcher CarFetcher - routing routing.ValueStore - namesys namesys.NameSystem - bstore blockstore.Blockstore - bsrv blockservice.BlockService + fetcher CarFetcher + blockFetcher exchange.Fetcher + routing routing.ValueStore + namesys namesys.NameSystem + bstore blockstore.Blockstore + bsrv blockservice.BlockService } -func NewGraphGatewayBackend(f CarFetcher, opts ...GraphGatewayOption) (*GraphGateway, error) { +func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts ...GraphGatewayOption) (*GraphGateway, error) { var compiledOptions gwOptions for _, o := range opts { if err := o(&compiledOptions); err != nil { @@ -118,10 +120,11 @@ func NewGraphGatewayBackend(f CarFetcher, opts ...GraphGatewayOption) (*GraphGat } return &GraphGateway{ - fetcher: f, - routing: vs, - namesys: ns, - bstore: bs, + fetcher: f, + blockFetcher: blockFetcher, + routing: vs, + namesys: ns, + bstore: bs, }, nil } @@ -141,9 +144,13 @@ func (api *GraphGateway) loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx con return nil, err } bstore := blockstore.NewBlockstore(ds) - exch := newInboundBlockExchange() - - doneWithFetcher := make(chan error, 1) + carFetchingExch := newInboundBlockExchange() + doneWithFetcher := make(chan struct{}, 1) + exch := &handoffExchange{ + startingExchange: carFetchingExch, + followupExchange: &blockFetcherExchWrapper{api.blockFetcher}, + handoffCh: doneWithFetcher, + } go func() { defer func() { @@ -151,7 +158,7 @@ func (api *GraphGateway) loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx con fmt.Println("Recovered fetcher error", r) } }() - doneWithFetcher <- api.fetcher.Fetch(ctx, path, func(resource string, reader io.Reader) error { + err := api.fetcher.Fetch(ctx, path, func(resource string, reader io.Reader) error { cr, err := car.NewCarReader(reader) if err != nil { return err @@ -167,11 +174,16 @@ func (api *GraphGateway) loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx con if err := bstore.Put(ctx, blk); err != nil { return err } - if err := exch.NotifyNewBlocks(ctx, blk); err != nil { + if err := carFetchingExch.NotifyNewBlocks(ctx, blk); err != nil { return err } } }) + if err != nil { + goLog.Error(err) + } + doneWithFetcher <- struct{}{} + close(doneWithFetcher) }() bserv := blockservice.New(bstore, exch) @@ -352,3 +364,119 @@ func (i *inboundBlockExchange) Close() error { } var _ exchange.Interface = (*inboundBlockExchange)(nil) + +type handoffExchange struct { + startingExchange, followupExchange exchange.Interface + handoffCh <-chan struct{} +} + +func (f *handoffExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + blkCh, err := f.startingExchange.GetBlocks(ctx, []cid.Cid{c}) + if err != nil { + return nil, err + } + blk, ok := <-blkCh + if ok { + return blk, nil + } + + select { + case <-f.handoffCh: + goLog.Infof("needed to use use a backup fetcher for cid %s", c) + return f.followupExchange.GetBlock(ctx, c) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (f *handoffExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { + blkCh, err := f.startingExchange.GetBlocks(ctx, cids) + if err != nil { + return nil, err + } + + retCh := make(chan blocks.Block) + + go func() { + cs := cid.NewSet() + for cs.Len() < len(cids) { + blk, ok := <-blkCh + if !ok { + break + } + select { + case retCh <- blk: + cs.Add(blk.Cid()) + case <-ctx.Done(): + } + } + + for cs.Len() < len(cids) { + select { + case <-ctx.Done(): + return + case <-f.handoffCh: + var newCidArr []cid.Cid + for _, c := range cids { + if !cs.Has(c) { + newCidArr = append(newCidArr, c) + } + } + goLog.Infof("needed to use use a backup fetcher for cids %v", newCidArr) + fch, err := f.followupExchange.GetBlocks(ctx, newCidArr) + if err != nil { + goLog.Error(fmt.Errorf("error getting blocks from followup exchange %w", err)) + return + } + for cs.Len() < len(cids) { + select { + case blk := <-fch: + select { + case retCh <- blk: + cs.Add(blk.Cid()) + case <-ctx.Done(): + return + } + } + } + } + } + }() + return retCh, nil +} + +func (f *handoffExchange) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { + err1 := f.startingExchange.NotifyNewBlocks(ctx, blocks...) + err2 := f.followupExchange.NotifyNewBlocks(ctx, blocks...) + return multierr.Combine(err1, err2) +} + +func (f *handoffExchange) Close() error { + err1 := f.startingExchange.Close() + err2 := f.followupExchange.Close() + return multierr.Combine(err1, err2) +} + +var _ exchange.Interface = (*handoffExchange)(nil) + +type blockFetcherExchWrapper struct { + f exchange.Fetcher +} + +func (b *blockFetcherExchWrapper) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + return b.f.GetBlock(ctx, c) +} + +func (b *blockFetcherExchWrapper) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan blocks.Block, error) { + return b.f.GetBlocks(ctx, cids) +} + +func (b *blockFetcherExchWrapper) NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error { + return nil +} + +func (b *blockFetcherExchWrapper) Close() error { + return nil +} + +var _ exchange.Interface = (*blockFetcherExchWrapper)(nil) From 763b3547664f2f7d8b3371442a5601cdcf6bfc8e Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 24 Mar 2023 07:20:14 -0400 Subject: [PATCH 10/27] gitignore: add .exe --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 56164e1..a96abb4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.coverprofile *.test *.orig +*.exe *~ .tarball From 7deef313157f71728d4868672539cdb7f6f77286 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 24 Mar 2023 07:21:04 -0400 Subject: [PATCH 11/27] looser CAR accept header Co-authored-by: Marcin Rataj --- backend/handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/handlers.go b/backend/handlers.go index c4def40..7ad4a82 100644 --- a/backend/handlers.go +++ b/backend/handlers.go @@ -100,7 +100,7 @@ func makeGatewayCARHandler(bsrv blockservice.BlockService, port int) (*http.Serv for _, header := range r.Header.Values("Accept") { for _, value := range strings.Split(header, ",") { accept := strings.TrimSpace(value) - if accept == "application/vnd.ipld.car" { + if strings.HasPrefix(accept, "application/vnd.ipld.car") { isCar = true break } From 7d4ae9ebb6aaf97b1b666edd5c0a55f5b1d71c0e Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 24 Mar 2023 07:28:28 -0400 Subject: [PATCH 12/27] switch magic numbers for graph gateway block cache Co-authored-by: Marcin Rataj --- lib/graph_gateway.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 5edda8b..894e1c2 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -109,7 +109,7 @@ func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts .. bs := compiledOptions.bs if compiledOptions.bs == nil { // Sets up a cache to store blocks in - cbs, err := NewCacheBlockStore(1000) + cbs, err := NewCacheBlockStore(DefaultCacheBlockStoreSize) if err != nil { return nil, err } From cf9c177854de8c80a232ca99709c1324ae6eac7d Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 24 Mar 2023 07:29:52 -0400 Subject: [PATCH 13/27] rename PubSub to BlockPubSub --- lib/graph_gateway.go | 4 ++-- lib/pubsub.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 894e1c2..2f08149 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -323,12 +323,12 @@ func (api *GraphGateway) GetDNSLinkRecord(ctx context.Context, hostname string) var _ gateway.IPFSBackend = (*GraphGateway)(nil) type inboundBlockExchange struct { - ps PubSub + ps BlockPubSub } func newInboundBlockExchange() *inboundBlockExchange { return &inboundBlockExchange{ - ps: NewPubSub(), + ps: NewBlockPubSub(), } } diff --git a/lib/pubsub.go b/lib/pubsub.go index 07d8299..b15d933 100644 --- a/lib/pubsub.go +++ b/lib/pubsub.go @@ -14,18 +14,18 @@ import ( const bufferSize = 16 -// PubSub is a simple interface for publishing blocks and being able to subscribe +// BlockPubSub is a simple interface for publishing blocks and being able to subscribe // for multihashes. It's used internally by an exchange to decouple receiving blocks // and actually providing them back to the GetBlocks caller. // Note: because multihashes are being requested and blocks returned the codecs could be anything -type PubSub interface { +type BlockPubSub interface { Publish(blocks ...blocks.Block) Subscribe(ctx context.Context, keys ...multihash.Multihash) <-chan blocks.Block Shutdown() } -// NewPubSub generates a new PubSub interface. -func NewPubSub() PubSub { +// NewBlockPubSub generates a new BlockPubSub interface. +func NewBlockPubSub() BlockPubSub { return &impl{ wrapped: *pubsub.New(bufferSize), closed: make(chan struct{}), From 6620f164586ebcbda9600e438dd5e4e1e8aae427 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 25 Mar 2023 02:04:29 +0100 Subject: [PATCH 14/27] fix: car fetch path redundant / caused 404 on some gateways --- blockstore_proxy.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/blockstore_proxy.go b/blockstore_proxy.go index 5f17450..e33d5a3 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -37,10 +37,11 @@ type proxyBlockStore struct { } func (ps *proxyBlockStore) Fetch(ctx context.Context, path string, cb lib.DataCallback) error { - u, err := url.Parse(fmt.Sprintf("%s/%s", ps.getRandomGatewayURL(), path)) + u, err := url.Parse(fmt.Sprintf("%s%s", ps.getRandomGatewayURL(), path)) if err != nil { return err } + goLog.Debugw("car fetch", "url", u) resp, err := ps.httpClient.Do(&http.Request{ Method: http.MethodGet, URL: u, @@ -102,6 +103,7 @@ func (ps *proxyBlockStore) fetch(ctx context.Context, c cid.Cid) (blocks.Block, if err != nil { return nil, err } + goLog.Debugw("raw fetch", "url", u) resp, err := ps.httpClient.Do(&http.Request{ Method: http.MethodGet, URL: u, From efd4f1b0b4840471d855230a0418da8775a9ba38 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 25 Mar 2023 03:03:29 +0100 Subject: [PATCH 15/27] docs: GRAPH_BACKEND Makes it easy to test both locally and on staging. --- blockstore_proxy.go | 2 +- docs/environment-variables.md | 19 +++++++++++++++++++ handlers.go | 7 ++++--- main.go | 23 ++++++++++++++++++----- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/blockstore_proxy.go b/blockstore_proxy.go index e33d5a3..2659aee 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -26,7 +26,7 @@ const ( EnvProxyGateway = "PROXY_GATEWAY_URL" DefaultProxyGateway = "http://127.0.0.1:8080" - DefaultKuboPRC = "http://127.0.0.1:5001" + DefaultKuboRPC = "http://127.0.0.1:5001" ) type proxyBlockStore struct { diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 2e52f1d..66424a7 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -5,6 +5,7 @@ - [Configuration](#configuration) - [`KUBO_RPC_URL`](#kubo_rpc_url) - [`BLOCK_CACHE_SIZE`](#block_cache_size) + - [`GRAPH_BACKEND`](#graph_backend) - [Proxy Backend](#proxy-backend) - [`PROXY_GATEWAY_URL`](#proxy_gateway_url) - [Saturn Backend](#saturn-backend) @@ -22,6 +23,8 @@ ### `KUBO_RPC_URL` +Default: see `DefaultKuboRPC` + Single URL or a comma separated list of RPC endpoints that provide `/api/v0` from Kubo. We use this as temporary solution for IPNS Record routing until [IPIP-351](https://github.com/ipfs/specs/pull/351) ships with Kubo 0.19, @@ -29,8 +32,24 @@ and we also redirect some legacy `/api/v0` commands that need to be handled on ` ### `BLOCK_CACHE_SIZE` +Default: see `DefaultCacheBlockStoreSize` + The size of in-memory [2Q cache](https://pkg.go.dev/github.com/hashicorp/golang-lru/v2#TwoQueueCache) with recently used and most requently used blocks. +### `GRAPH_BACKEND` + +Default: `false` + +When set to `true`, requests to backend will use +`?format=car&depth=..&bytes=..` in addition to `?format=raw` to reduce the +number of round trips. + +This is an experimental feature that depends on `&depth=..&bytes=..` +parameters. Currently only `https://l1s.strn.pl` supports it, but our +intention is to standardize it and add it to the +[trustless gateway spec](https://specs.ipfs.tech/http-gateways/trustless-gateway/) +in the near feature. + ## Proxy Backend ### `PROXY_GATEWAY_URL` diff --git a/handlers.go b/handlers.go index 23136b1..7295a04 100644 --- a/handlers.go +++ b/handlers.go @@ -2,12 +2,13 @@ package main import ( "fmt" - "github.com/ipfs/bifrost-gateway/lib" "math/rand" "net/http" "strconv" "time" + "github.com/ipfs/bifrost-gateway/lib" + "github.com/filecoin-saturn/caboose" "github.com/ipfs/go-blockservice" bstore "github.com/ipfs/go-ipfs-blockstore" @@ -41,13 +42,13 @@ func withRequestLogger(next http.Handler) http.Handler { }) } -func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockCacheSize int, cdns *cachedDNS, useGraphGatewayBackend bool) (*http.Server, error) { +func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockCacheSize int, cdns *cachedDNS, useGraphBackend bool) (*http.Server, error) { // Sets up the routing system, which will proxy the IPNS routing requests to the given gateway. routing := newProxyRouting(kuboRPC, cdns) var gwAPI gateway.IPFSBackend var err error - if !useGraphGatewayBackend { + if !useGraphBackend { // Sets up an exchange based on the given Block Store exch, err := newExchange(bs) if err != nil { diff --git a/main.go b/main.go index cfc49bc..b2a139b 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( _ "embed" "errors" "fmt" - "github.com/ipfs/bifrost-gateway/lib" "log" "net/http" "os" @@ -13,6 +12,8 @@ import ( "strings" "sync" + "github.com/ipfs/bifrost-gateway/lib" + blockstore "github.com/ipfs/go-ipfs-blockstore" golog "github.com/ipfs/go-log/v2" "github.com/spf13/cobra" @@ -30,12 +31,12 @@ func main() { const ( EnvKuboRPC = "KUBO_RPC_URL" EnvBlockCacheSize = "BLOCK_CACHE_SIZE" + EnvGraphBackend = "GRAPH_BACKEND" ) func init() { rootCmd.Flags().Int("gateway-port", 8081, "gateway port") rootCmd.Flags().Int("metrics-port", 8041, "metrics port") - rootCmd.Flags().Bool("graph-gateway", false, "use a graph fetching based gateway") } var rootCmd = &cobra.Command{ @@ -49,18 +50,22 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, // Get flags. gatewayPort, _ := cmd.Flags().GetInt("gateway-port") metricsPort, _ := cmd.Flags().GetInt("metrics-port") - useGraphGateway, _ := cmd.Flags().GetBool("graph-gateway") // Get env variables. saturnOrchestrator := getEnv(EnvSaturnOrchestrator, "") proxyGateway := getEnvs(EnvProxyGateway, "") - kuboRPC := getEnvs(EnvKuboRPC, DefaultKuboPRC) + kuboRPC := getEnvs(EnvKuboRPC, DefaultKuboRPC) blockCacheSize, err := getEnvInt(EnvBlockCacheSize, lib.DefaultCacheBlockStoreSize) if err != nil { return err } + useGraphBackend, err := getEnvBool(EnvGraphBackend, false) + if err != nil { + return err + } + log.Printf("Starting %s %s", name, version) cdns := newCachedDNS(dnsCacheRefreshInterval) @@ -88,7 +93,7 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, log.Fatalf("Unable to start. bifrost-gateway requires either PROXY_GATEWAY_URL or STRN_ORCHESTRATOR_URL to be set.\n\nRead docs at https://github.com/ipfs/bifrost-gateway/blob/main/docs/environment-variables.md\n\n") } - gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, gatewayPort, blockCacheSize, cdns, useGraphGateway) + gatewaySrv, err := makeGatewayHandler(bs, kuboRPC, gatewayPort, blockCacheSize, cdns, useGraphBackend) if err != nil { return err } @@ -165,3 +170,11 @@ func getEnvInt(key string, defaultValue int) (int, error) { } return strconv.Atoi(value) } + +func getEnvBool(key string, defaultValue bool) (bool, error) { + value := os.Getenv(key) + if value == "" { + return defaultValue, nil + } + return strconv.ParseBool(value) +} From 2d8f9faab5696101a799c877d5bfdd11f9e0ec9c Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 27 Mar 2023 15:39:27 -0400 Subject: [PATCH 16/27] feat: add debugging endpoints --- handlers.go | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++- main.go | 2 +- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/handlers.go b/handlers.go index 7295a04..9b6c77c 100644 --- a/handlers.go +++ b/handlers.go @@ -4,9 +4,12 @@ import ( "fmt" "math/rand" "net/http" + "runtime" "strconv" "time" + _ "net/http/pprof" + "github.com/ipfs/bifrost-gateway/lib" "github.com/filecoin-saturn/caboose" @@ -18,7 +21,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" ) -func makeMetricsHandler(port int) (*http.Server, error) { +func makeMetricsAndDebuggingHandler(port int) (*http.Server, error) { mux := http.NewServeMux() gatherers := prometheus.Gatherers{ @@ -27,6 +30,11 @@ func makeMetricsHandler(port int) (*http.Server, error) { } options := promhttp.HandlerOpts{} mux.Handle("/debug/metrics/prometheus", promhttp.HandlerFor(gatherers, options)) + mux.Handle("/debug/vars", http.DefaultServeMux) + mux.Handle("/debug/pprof/", http.DefaultServeMux) + mux.Handle("/debug/stack", http.DefaultServeMux) + MutexFractionOption("/debug/pprof-mutex/", mux) + BlockProfileRateOption("/debug/pprof-block/", mux) return &http.Server{ Handler: mux, @@ -203,3 +211,65 @@ func newKuboRPCHandler(endpoints []string) http.Handler { return mux } + +// MutexFractionOption allows to set runtime.SetMutexProfileFraction via HTTP +// using POST request with parameter 'fraction'. +func MutexFractionOption(path string, mux *http.ServeMux) *http.ServeMux { + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "only POST allowed", http.StatusMethodNotAllowed) + return + } + if err := r.ParseForm(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + asfr := r.Form.Get("fraction") + if len(asfr) == 0 { + http.Error(w, "parameter 'fraction' must be set", http.StatusBadRequest) + return + } + + fr, err := strconv.Atoi(asfr) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + runtime.SetMutexProfileFraction(fr) + }) + + return mux +} + +// BlockProfileRateOption allows to set runtime.SetBlockProfileRate via HTTP +// using POST request with parameter 'rate'. +// The profiler tries to sample 1 event every nanoseconds. +// If rate == 1, then the profiler samples every blocking event. +// To disable, set rate = 0. +func BlockProfileRateOption(path string, mux *http.ServeMux) *http.ServeMux { + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "only POST allowed", http.StatusMethodNotAllowed) + return + } + if err := r.ParseForm(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + rateStr := r.Form.Get("rate") + if len(rateStr) == 0 { + http.Error(w, "parameter 'rate' must be set", http.StatusBadRequest) + return + } + + rate, err := strconv.Atoi(rateStr) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + runtime.SetBlockProfileRate(rate) + }) + return mux +} diff --git a/main.go b/main.go index b2a139b..62c9302 100644 --- a/main.go +++ b/main.go @@ -98,7 +98,7 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, return err } - metricsSrv, err := makeMetricsHandler(metricsPort) + metricsSrv, err := makeMetricsAndDebuggingHandler(metricsPort) if err != nil { return err } From 04e0bc4842778c4cf3b5185da4b92d068ed7c8e1 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 27 Mar 2023 15:40:08 -0400 Subject: [PATCH 17/27] feat(graph-gateway): switch to shared blockstore --- lib/graph_gateway.go | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 2f08149..c7b3b5d 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -7,7 +7,6 @@ import ( "github.com/filecoin-saturn/caboose" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" - leveldb "github.com/ipfs/go-ds-leveldb" blockstore "github.com/ipfs/go-ipfs-blockstore" exchange "github.com/ipfs/go-ipfs-exchange-interface" "github.com/ipfs/go-libipfs/blocks" @@ -138,12 +137,8 @@ Implementation iteration plan: 5. Don't redo the last segment fully if it's part of a UnixFS file and we can do range requests */ -func (api *GraphGateway) loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx context.Context, path string) (gateway.IPFSBackend, error) { - ds, err := leveldb.NewDatastore("", nil) - if err != nil { - return nil, err - } - bstore := blockstore.NewBlockstore(ds) +func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx context.Context, path string) (gateway.IPFSBackend, error) { + bstore := api.bstore carFetchingExch := newInboundBlockExchange() doneWithFetcher := make(chan struct{}, 1) exch := &handoffExchange{ @@ -196,7 +191,7 @@ func (api *GraphGateway) loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx con } func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { - blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") + blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -205,7 +200,7 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath) (g func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePath, getRange ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) { // TODO: actually implement ranges - blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") + blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -213,7 +208,7 @@ func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePat } func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=all") + blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=all") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -221,7 +216,7 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) } func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { - blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") + blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -229,7 +224,7 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat } func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&bytes=0:1023") + blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&bytes=0:1023") if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -237,7 +232,7 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( } func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { - blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") + blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") if err != nil { return gateway.ContentPathMetadata{}, err } @@ -245,7 +240,7 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable } func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) { - blkgw, err := api.loadRequestIntoMemoryBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car") + blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car") if err != nil { return gateway.ContentPathMetadata{}, nil, nil, err } From d8bdf201d06f5e61a219f7b841ece0170763d160 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 27 Mar 2023 16:49:13 -0400 Subject: [PATCH 18/27] feat(graph-gateway): shared blockstore cache size is configurable --- handlers.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/handlers.go b/handlers.go index 9b6c77c..8aefebe 100644 --- a/handlers.go +++ b/handlers.go @@ -54,8 +54,16 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC // Sets up the routing system, which will proxy the IPNS routing requests to the given gateway. routing := newProxyRouting(kuboRPC, cdns) + // Sets up a cache to store blocks in + cacheBlockStore, err := lib.NewCacheBlockStore(blockCacheSize) + if err != nil { + return nil, err + } + + // Set up support for identity hashes (https://github.com/ipfs/bifrost-gateway/issues/38) + cacheBlockStore = bstore.NewIdStore(cacheBlockStore) + var gwAPI gateway.IPFSBackend - var err error if !useGraphBackend { // Sets up an exchange based on the given Block Store exch, err := newExchange(bs) @@ -63,15 +71,6 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC return nil, err } - // Sets up a cache to store blocks in - cacheBlockStore, err := lib.NewCacheBlockStore(blockCacheSize) - if err != nil { - return nil, err - } - - // Set up support for identity hashes (https://github.com/ipfs/bifrost-gateway/issues/38) - cacheBlockStore = bstore.NewIdStore(cacheBlockStore) - // Sets up a blockservice which tries the cache and falls back to the exchange blockService := blockservice.New(cacheBlockStore, exch) @@ -87,7 +86,7 @@ func makeGatewayHandler(bs bstore.Blockstore, kuboRPC []string, port int, blockC return nil, err } - gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), exch, lib.WithValueStore(routing)) + gwAPI, err = lib.NewGraphGatewayBackend(bs.(lib.CarFetcher), exch, lib.WithValueStore(routing), lib.WithBlockstore(cacheBlockStore)) if err != nil { return nil, err } From 6c4ccc980b083c1a2c1e80f6e5dd7d231ee7ec18 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 29 Mar 2023 11:47:25 -0400 Subject: [PATCH 19/27] fix(graph-gateway): falling back to block requests, and use shared notifier --- lib/graph_gateway.go | 145 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 127 insertions(+), 18 deletions(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index c7b3b5d..86251d1 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -26,6 +26,8 @@ import ( "go.uber.org/multierr" "io" "net/http" + "runtime" + "sync" ) // type DataCallback = func(resource string, reader io.Reader) error @@ -69,6 +71,10 @@ func WithBlockstore(bs blockstore.Blockstore) GraphGatewayOption { type GraphGatewayOption func(gwOptions *gwOptions) error +type Notifier interface { + NotifyNewBlocks(ctx context.Context, blocks ...blocks.Block) error +} + type GraphGateway struct { fetcher CarFetcher blockFetcher exchange.Fetcher @@ -76,6 +82,9 @@ type GraphGateway struct { namesys namesys.NameSystem bstore blockstore.Blockstore bsrv blockservice.BlockService + + lk sync.RWMutex + notifiers map[Notifier]struct{} } func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts ...GraphGatewayOption) (*GraphGateway, error) { @@ -124,6 +133,7 @@ func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts .. routing: vs, namesys: ns, bstore: bs, + notifiers: make(map[Notifier]struct{}), }, nil } @@ -137,7 +147,7 @@ Implementation iteration plan: 5. Don't redo the last segment fully if it's part of a UnixFS file and we can do range requests */ -func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx context.Context, path string) (gateway.IPFSBackend, error) { +func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx context.Context, path string) (gateway.IPFSBackend, func(), error) { bstore := api.bstore carFetchingExch := newInboundBlockExchange() doneWithFetcher := make(chan struct{}, 1) @@ -147,6 +157,10 @@ func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx con handoffCh: doneWithFetcher, } + api.lk.Lock() + api.notifiers[exch] = struct{}{} + api.lk.Unlock() + go func() { defer func() { if r := recover(); r != nil { @@ -169,14 +183,15 @@ func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx con if err := bstore.Put(ctx, blk); err != nil { return err } - if err := carFetchingExch.NotifyNewBlocks(ctx, blk); err != nil { - return err - } + api.notifyAllOngoingRequests(ctx, blk) } }) if err != nil { goLog.Error(err) } + if err := carFetchingExch.Close(); err != nil { + goLog.Error(err) + } doneWithFetcher <- struct{}{} close(doneWithFetcher) }() @@ -184,66 +199,160 @@ func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx con bserv := blockservice.New(bstore, exch) blkgw, err := gateway.NewBlocksGateway(bserv) if err != nil { - return nil, err + return nil, nil, err } - return blkgw, nil + return blkgw, func() { + api.lk.Lock() + delete(api.notifiers, exch) + api.lk.Unlock() + }, nil +} + +func (api *GraphGateway) notifyAllOngoingRequests(ctx context.Context, blks ...blocks.Block) { + api.lk.RLock() + for n := range api.notifiers { + err := n.NotifyNewBlocks(ctx, blks...) + if err != nil { + goLog.Error(fmt.Errorf("notifyAllOngoingRequests failed: %w", err)) + } + } + api.lk.RUnlock() +} + +type fileCloseWrapper struct { + files.File + closeFn func() +} + +func (w *fileCloseWrapper) Close() error { + w.closeFn() + return w.File.Close() +} + +type dirCloseWrapper struct { + files.Directory + closeFn func() +} + +func (w *dirCloseWrapper) Close() error { + w.closeFn() + return w.Directory.Close() +} + +func wrapNodeWithClose[T files.Node](node T, closeFn func()) (T, error) { + var genericNode files.Node = node + switch n := genericNode.(type) { + case files.File: + var f files.File = &fileCloseWrapper{n, closeFn} + return f.(T), nil + case files.Directory: + var d files.Directory = &dirCloseWrapper{n, closeFn} + return d.(T), nil + case *files.Symlink: + closeFn() + return node, nil + default: + closeFn() + var zeroType T + return zeroType, fmt.Errorf("unsupported node type") + } } func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { - blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") if err != nil { return gateway.ContentPathMetadata{}, nil, err } - return blkgw.Get(ctx, path) + md, gr, err := blkgw.Get(ctx, path) + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + //TODO: interfaces here aren't good enough so we're getting around the problem this way + runtime.SetFinalizer(gr, func(_ *gateway.GetResponse) { closeFn() }) + return md, gr, err } func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePath, getRange ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) { // TODO: actually implement ranges - blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + md, f, err := blkgw.GetRange(ctx, path) + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + f, err = wrapNodeWithClose(f, closeFn) if err != nil { return gateway.ContentPathMetadata{}, nil, err } - return blkgw.GetRange(ctx, path) + return md, f, nil } func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=all") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=all") if err != nil { return gateway.ContentPathMetadata{}, nil, err } - return blkgw.GetAll(ctx, path) + md, f, err := blkgw.GetAll(ctx, path) + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + f, err = wrapNodeWithClose(f, closeFn) + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + return md, f, nil } func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { - blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + md, f, err := blkgw.GetBlock(ctx, path) + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + f, err = wrapNodeWithClose(f, closeFn) if err != nil { return gateway.ContentPathMetadata{}, nil, err } - return blkgw.GetBlock(ctx, path) + return md, f, nil } func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&bytes=0:1023") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&bytes=0:1023") + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + md, f, err := blkgw.Head(ctx, path) + if err != nil { + return gateway.ContentPathMetadata{}, nil, err + } + f, err = wrapNodeWithClose(f, closeFn) if err != nil { return gateway.ContentPathMetadata{}, nil, err } - return blkgw.Head(ctx, path) + return md, f, nil } func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { - blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") if err != nil { return gateway.ContentPathMetadata{}, err } + defer closeFn() return blkgw.ResolvePath(ctx, path) } func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) { - blkgw, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car") + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car") if err != nil { return gateway.ContentPathMetadata{}, nil, nil, err } + defer closeFn() return blkgw.GetCAR(ctx, path) } From 2044635c1e2bc0f8bd55fc193bea9f99da331c9c Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 29 Mar 2023 12:54:09 -0400 Subject: [PATCH 20/27] fix errors with channel closure --- lib/graph_gateway.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 002f6e8..2c06815 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -18,6 +18,7 @@ import ( ipfspath "github.com/ipfs/boxo/path" "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" @@ -437,10 +438,13 @@ func newInboundBlockExchange() *inboundBlockExchange { } func (i *inboundBlockExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { - blk := <-i.ps.Subscribe(ctx, c.Hash()) + blk, more := <-i.ps.Subscribe(ctx, c.Hash()) if err := ctx.Err(); err != nil { return nil, err } + if !more { + return nil, format.ErrNotFound{Cid: c} + } return blk, nil } @@ -534,7 +538,10 @@ func (f *handoffExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan } for cs.Len() < len(cids) { select { - case blk := <-fch: + case blk, ok := <-fch: + if !ok { + return + } select { case retCh <- blk: cs.Add(blk.Cid()) From 0a92fd01653f4983ad5e2483b3bb4ff796192a4b Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 29 Mar 2023 13:00:18 -0400 Subject: [PATCH 21/27] staticcheck fixes --- lib/graph_gateway.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 2c06815..ad9991b 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -82,7 +82,6 @@ type GraphGateway struct { routing routing.ValueStore namesys namesys.NameSystem bstore blockstore.Blockstore - bsrv blockservice.BlockService lk sync.RWMutex notifiers map[Notifier]struct{} @@ -244,15 +243,15 @@ func (w *dirCloseWrapper) Close() error { func wrapNodeWithClose[T files.Node](node T, closeFn func()) (T, error) { var genericNode files.Node = node switch n := genericNode.(type) { + case *files.Symlink: + closeFn() + return node, nil case files.File: var f files.File = &fileCloseWrapper{n, closeFn} return f.(T), nil case files.Directory: var d files.Directory = &dirCloseWrapper{n, closeFn} return d.(T), nil - case *files.Symlink: - closeFn() - return node, nil default: closeFn() var zeroType T @@ -537,17 +536,15 @@ func (f *handoffExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan return } for cs.Len() < len(cids) { + blk, ok := <-fch + if !ok { + return + } select { - case blk, ok := <-fch: - if !ok { - return - } - select { - case retCh <- blk: - cs.Add(blk.Cid()) - case <-ctx.Done(): - return - } + case retCh <- blk: + cs.Add(blk.Cid()) + case <-ctx.Done(): + return } } } From f06dc972af600e0ebaa9779d27edf9528b1c1f06 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 29 Mar 2023 13:08:43 -0400 Subject: [PATCH 22/27] backend staticcheck fixes and add Server header --- backend/handlers.go | 4 ++-- backend/main.go | 30 ------------------------------ 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/backend/handlers.go b/backend/handlers.go index ae1180c..23c5d85 100644 --- a/backend/handlers.go +++ b/backend/handlers.go @@ -130,11 +130,11 @@ func makeGatewayCARHandler(bsrv blockservice.BlockService, port int) (*http.Serv const immutableCacheControl = "public, max-age=29030400, immutable" // immutable! CACHE ALL THE THINGS, FOREVER! wolololol w.Header().Set("Cache-Control", immutableCacheControl) + w.Header().Set("Server", userAgent) // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) io.Copy(w, carStream) - return }) // Creates metrics handler for total response size. Matches the same metrics @@ -591,11 +591,11 @@ func makeGatewayBlockHandler(bsrv blockservice.BlockService, port int) (*http.Se const immutableCacheControl = "public, max-age=29030400, immutable" // immutable! CACHE ALL THE THINGS, FOREVER! wolololol w.Header().Set("Cache-Control", immutableCacheControl) + w.Header().Set("Server", userAgent) // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) http.ServeContent(w, r, c.String()+".bin", noModtime, bytes.NewReader(blk.RawData())) - return }) // Creates metrics handler for total response size. Matches the same metrics diff --git a/backend/main.go b/backend/main.go index 4ed3662..0cf7eec 100644 --- a/backend/main.go +++ b/backend/main.go @@ -19,8 +19,6 @@ import ( "net/http" "os" "os/signal" - "strconv" - "strings" "sync" ) @@ -146,31 +144,3 @@ var rootCmd = &cobra.Command{ return nil }, } - -func getEnv(key, defaultValue string) string { - value := os.Getenv(key) - if value == "" { - return defaultValue - } - return value -} - -func getEnvs(key, defaultValue string) []string { - value := os.Getenv(key) - if value == "" { - if defaultValue == "" { - return []string{} - } - value = defaultValue - } - value = strings.TrimSpace(value) - return strings.Split(value, ",") -} - -func getEnvInt(key string, defaultValue int) (int, error) { - value := os.Getenv(key) - if value == "" { - return defaultValue, nil - } - return strconv.Atoi(value) -} From ba35d747846bb7cf5f6840a480182262aeeb71cb Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 29 Mar 2023 20:09:15 +0200 Subject: [PATCH 23/27] fix: car request timeout and friends 19s was way too low: https://github.com/filecoin-saturn/caboose/pull/68/commits/aaf9ff90a18d76be96c213ab037625dffed5c6ef --- backend/main.go | 13 +++++++------ blockstore_caboose.go | 10 +++++----- go.mod | 2 +- go.sum | 4 ++-- lib/blockstore_cache.go | 12 ++++++------ lib/graph_gateway.go | 26 +++++++++++++++----------- 6 files changed, 36 insertions(+), 31 deletions(-) diff --git a/backend/main.go b/backend/main.go index 0cf7eec..1d3d0d3 100644 --- a/backend/main.go +++ b/backend/main.go @@ -3,6 +3,12 @@ package main import ( "errors" "fmt" + "log" + "net/http" + "os" + "os/signal" + "sync" + "github.com/ipfs/bifrost-gateway/lib" "github.com/ipfs/boxo/bitswap/client" "github.com/ipfs/boxo/bitswap/network" @@ -15,14 +21,9 @@ import ( "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/routing" "github.com/spf13/cobra" - "log" - "net/http" - "os" - "os/signal" - "sync" ) -var goLog = golog.Logger("bifrost-gateway-backend") +var goLog = golog.Logger("test-backend") func main() { if err := rootCmd.Execute(); err != nil { diff --git a/blockstore_caboose.go b/blockstore_caboose.go index 19a376a..5021f4e 100644 --- a/blockstore_caboose.go +++ b/blockstore_caboose.go @@ -41,7 +41,7 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS) } saturnOrchestratorClient := &http.Client{ - Timeout: caboose.DefaultSaturnRequestTimeout, + Timeout: caboose.DefaultSaturnOrchestratorRequestTimeout, Transport: &customTransport{ RoundTripper: &http.Transport{ DialContext: cdns.dialWithCachedDNS, @@ -50,7 +50,7 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS) } saturnLoggerClient := &http.Client{ - Timeout: caboose.DefaultSaturnRequestTimeout, + Timeout: caboose.DefaultSaturnLoggerRequestTimeout, Transport: &customTransport{ AuthorizationBearerToken: os.Getenv(EnvSaturnLoggerSecret), RoundTripper: &http.Transport{ @@ -60,7 +60,7 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS) } saturnRetrievalClient := &http.Client{ - Timeout: caboose.DefaultSaturnRequestTimeout, + Timeout: caboose.DefaultSaturnCarRequestTimeout, Transport: &customTransport{ RoundTripper: &http.Transport{ // Increasing concurrency defaults from http.DefaultTransport @@ -93,10 +93,10 @@ func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS) LoggingEndpoint: *loggURL, LoggingClient: saturnLoggerClient, - LoggingInterval: 5 * time.Second, + LoggingInterval: caboose.DefaultLoggingInterval, DoValidation: true, - PoolRefresh: 5 * time.Minute, + PoolRefresh: caboose.DefaultPoolRefreshInterval, SaturnClient: saturnRetrievalClient, }) } diff --git a/go.mod b/go.mod index 800c55c..ab03fc2 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/cskr/pubsub v1.0.2 - github.com/filecoin-saturn/caboose v0.0.0-20230329141039-b4cc70873ccd + github.com/filecoin-saturn/caboose v0.0.0-20230329185035-5b37545e2a41 github.com/gogo/protobuf v1.3.2 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/ipfs/boxo v0.8.0-rc2.0.20230329082438-360b031ed895 diff --git a/go.sum b/go.sum index 246de2b..daa033a 100644 --- a/go.sum +++ b/go.sum @@ -63,8 +63,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/filecoin-saturn/caboose v0.0.0-20230329141039-b4cc70873ccd h1:+Je9eB4IlUAG/NP8seXFC5vD+hPM6mYMN4dOEQESSZI= -github.com/filecoin-saturn/caboose v0.0.0-20230329141039-b4cc70873ccd/go.mod h1:u8END6RLUG9ZWudtVHAOOg7ptGiAgKjf/QnNOUOC160= +github.com/filecoin-saturn/caboose v0.0.0-20230329185035-5b37545e2a41 h1:CRERrZrpsrSwWYyBzlQJZUeh9gy8LaOukw7E6dMgd3E= +github.com/filecoin-saturn/caboose v0.0.0-20230329185035-5b37545e2a41/go.mod h1:u8END6RLUG9ZWudtVHAOOg7ptGiAgKjf/QnNOUOC160= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= diff --git a/lib/blockstore_cache.go b/lib/blockstore_cache.go index 10fe85e..3bf63f8 100644 --- a/lib/blockstore_cache.go +++ b/lib/blockstore_cache.go @@ -8,7 +8,7 @@ import ( format "github.com/ipfs/go-ipld-format" blockstore "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/go-block-format" + blocks "github.com/ipfs/go-block-format" golog "github.com/ipfs/go-log/v2" lru "github.com/hashicorp/golang-lru/v2" @@ -19,7 +19,7 @@ import ( const DefaultCacheBlockStoreSize = 1024 -var goLog = golog.Logger("cache-blockstore") +var cacheLog = golog.Logger("bifrost-gateway:cache-blockstore") func NewCacheBlockStore(size int) (blockstore.Blockstore, error) { c, err := lru.New2Q[string, []byte](size) @@ -80,16 +80,16 @@ func (l *cacheBlockStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, err blkData, found := l.cache.Get(string(c.Hash())) if !found { - if goLog.Level().Enabled(zapcore.DebugLevel) { - goLog.Debugw("block not found in cache", "cid", c.String()) + if cacheLog.Level().Enabled(zapcore.DebugLevel) { + cacheLog.Debugw("block not found in cache", "cid", c.String()) } return nil, format.ErrNotFound{Cid: c} } // It's a HIT! l.cacheHitsMetric.Add(1) - if goLog.Level().Enabled(zapcore.DebugLevel) { - goLog.Debugw("block found in cache", "cid", c.String()) + if cacheLog.Level().Enabled(zapcore.DebugLevel) { + cacheLog.Debugw("block found in cache", "cid", c.String()) } if l.rehash.Load() { diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index ad9991b..7b1c1aa 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -4,6 +4,11 @@ import ( "context" "errors" "fmt" + "io" + "net/http" + "runtime" + "sync" + "github.com/filecoin-saturn/caboose" "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" @@ -16,21 +21,20 @@ import ( "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/namesys/resolve" ipfspath "github.com/ipfs/boxo/path" - "github.com/ipfs/go-block-format" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" + golog "github.com/ipfs/go-log/v2" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" "go.uber.org/multierr" - "io" - "net/http" - "runtime" - "sync" ) +var graphLog = golog.Logger("bifrost-gateway:graph-backend") + // type DataCallback = func(resource string, reader io.Reader) error // TODO: Don't use a caboose type, perhaps ask them to use a type alias instead of a type type DataCallback = caboose.DataCallback @@ -187,10 +191,10 @@ func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx con } }) if err != nil { - goLog.Error(err) + graphLog.Error(err) } if err := carFetchingExch.Close(); err != nil { - goLog.Error(err) + graphLog.Error(err) } doneWithFetcher <- struct{}{} close(doneWithFetcher) @@ -214,7 +218,7 @@ func (api *GraphGateway) notifyAllOngoingRequests(ctx context.Context, blks ...b for n := range api.notifiers { err := n.NotifyNewBlocks(ctx, blks...) if err != nil { - goLog.Error(fmt.Errorf("notifyAllOngoingRequests failed: %w", err)) + graphLog.Error(fmt.Errorf("notifyAllOngoingRequests failed: %w", err)) } } api.lk.RUnlock() @@ -489,7 +493,7 @@ func (f *handoffExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block select { case <-f.handoffCh: - goLog.Infof("needed to use use a backup fetcher for cid %s", c) + graphLog.Infof("needed to use use a backup fetcher for cid %s", c) return f.followupExchange.GetBlock(ctx, c) case <-ctx.Done(): return nil, ctx.Err() @@ -529,10 +533,10 @@ func (f *handoffExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan newCidArr = append(newCidArr, c) } } - goLog.Infof("needed to use use a backup fetcher for cids %v", newCidArr) + graphLog.Infof("needed to use use a backup fetcher for cids %v", newCidArr) fch, err := f.followupExchange.GetBlocks(ctx, newCidArr) if err != nil { - goLog.Error(fmt.Errorf("error getting blocks from followup exchange %w", err)) + graphLog.Error(fmt.Errorf("error getting blocks from followup exchange %w", err)) return } for cs.Len() < len(cids) { From fdc0d5ab9853a5f822ec23d96eec0a6aa8afbf97 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 29 Mar 2023 21:25:01 +0200 Subject: [PATCH 24/27] chore: rename to test-backend --- {backend => test-backend}/handlers.go | 0 {backend => test-backend}/main.go | 4 ++-- {backend => test-backend}/version.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename {backend => test-backend}/handlers.go (100%) rename {backend => test-backend}/main.go (96%) rename {backend => test-backend}/version.go (94%) diff --git a/backend/handlers.go b/test-backend/handlers.go similarity index 100% rename from backend/handlers.go rename to test-backend/handlers.go diff --git a/backend/main.go b/test-backend/main.go similarity index 96% rename from backend/main.go rename to test-backend/main.go index 1d3d0d3..0a31870 100644 --- a/backend/main.go +++ b/test-backend/main.go @@ -36,14 +36,14 @@ func init() { rootCmd.Flags().Int("gateway-port", 8082, "gateway port") rootCmd.Flags().Int("metrics-port", 8042, "metrics port") rootCmd.Flags().String("car-blockstore", "", "a CAR file to use for serving data instead of network requests") - golog.SetLogLevel("bifrost-gateway-backend", "debug") + golog.SetLogLevel("test-backend", "debug") } var rootCmd = &cobra.Command{ Use: name, Version: version, CompletionOptions: cobra.CompletionOptions{DisableDefaultCmd: true}, - Short: "IPFS Gateway backend implementation for https://github.com/protocol/bifrost-infra", + Short: "Test Gateway backend for https://github.com/ipfs/bifrost-gateway (EXPERIMENTAL)", RunE: func(cmd *cobra.Command, args []string) error { // Get flags. gatewayPort, _ := cmd.Flags().GetInt("gateway-port") diff --git a/backend/version.go b/test-backend/version.go similarity index 94% rename from backend/version.go rename to test-backend/version.go index ba6d080..318d7f5 100644 --- a/backend/version.go +++ b/test-backend/version.go @@ -5,7 +5,7 @@ import ( "time" ) -var name = "bifrost-gateway-backend" +var name = "test-backend" var version = buildVersion() var userAgent = name + "/" + version From 69b5a4938c747f99f8a2514117c01f5df96fe19a Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 31 Mar 2023 04:27:45 +0200 Subject: [PATCH 25/27] feat: IPFSBackend and GraphGateway metrics - IPFSBackend metrics from https://github.com/ipfs/boxo/pull/245 - GraphGateway metrics from https://github.com/ipfs/bifrost-gateway/pull/61 - Version for tracking rollouts --- go.mod | 4 +- go.sum | 8 +-- lib/blockstore_cache.go | 2 +- lib/graph_gateway.go | 141 ++++++++++++++++++++++++++++++++++++---- main.go | 1 + version.go | 14 ++++ 6 files changed, 150 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index ab03fc2..770fbec 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.19 require ( github.com/cskr/pubsub v1.0.2 - github.com/filecoin-saturn/caboose v0.0.0-20230329185035-5b37545e2a41 + github.com/filecoin-saturn/caboose v0.0.0-20230329203940-7c08345c244d github.com/gogo/protobuf v1.3.2 github.com/hashicorp/golang-lru/v2 v2.0.1 - github.com/ipfs/boxo v0.8.0-rc2.0.20230329082438-360b031ed895 + github.com/ipfs/boxo v0.8.0-rc3.0.20230331021433-e49cc43d95ad github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.0 github.com/ipfs/go-ipld-format v0.4.0 diff --git a/go.sum b/go.sum index daa033a..0e576a6 100644 --- a/go.sum +++ b/go.sum @@ -63,8 +63,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/filecoin-saturn/caboose v0.0.0-20230329185035-5b37545e2a41 h1:CRERrZrpsrSwWYyBzlQJZUeh9gy8LaOukw7E6dMgd3E= -github.com/filecoin-saturn/caboose v0.0.0-20230329185035-5b37545e2a41/go.mod h1:u8END6RLUG9ZWudtVHAOOg7ptGiAgKjf/QnNOUOC160= +github.com/filecoin-saturn/caboose v0.0.0-20230329203940-7c08345c244d h1:ok/4Ffp2ETy6l5kBRUumvxzGz/Hxqq2iTfmGLslZbBc= +github.com/filecoin-saturn/caboose v0.0.0-20230329203940-7c08345c244d/go.mod h1:u8END6RLUG9ZWudtVHAOOg7ptGiAgKjf/QnNOUOC160= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= @@ -186,8 +186,8 @@ github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7P github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.0-rc2.0.20230329082438-360b031ed895 h1:bh+8xMBQSOnieUSg7qToTFhPpf4Oc8QkvPSRmQSrnpc= -github.com/ipfs/boxo v0.8.0-rc2.0.20230329082438-360b031ed895/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= +github.com/ipfs/boxo v0.8.0-rc3.0.20230331021433-e49cc43d95ad h1:olq3866OLC+aCW3hImmUtze0Nh8lbFfciTSeq09xfQU= +github.com/ipfs/boxo v0.8.0-rc3.0.20230331021433-e49cc43d95ad/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/lib/blockstore_cache.go b/lib/blockstore_cache.go index 3bf63f8..7e986a6 100644 --- a/lib/blockstore_cache.go +++ b/lib/blockstore_cache.go @@ -19,7 +19,7 @@ import ( const DefaultCacheBlockStoreSize = 1024 -var cacheLog = golog.Logger("bifrost-gateway:cache-blockstore") +var cacheLog = golog.Logger("cache-blockstore") func NewCacheBlockStore(size int) (blockstore.Blockstore, error) { c, err := lru.New2Q[string, []byte](size) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 7b1c1aa..0df8059 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -7,6 +7,8 @@ import ( "io" "net/http" "runtime" + "strconv" + "strings" "sync" "github.com/filecoin-saturn/caboose" @@ -30,10 +32,11 @@ import ( "github.com/libp2p/go-libp2p/core/routing" "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/multierr" ) -var graphLog = golog.Logger("bifrost-gateway:graph-backend") +var graphLog = golog.Logger("graph-gateway") // type DataCallback = func(resource string, reader io.Reader) error // TODO: Don't use a caboose type, perhaps ask them to use a type alias instead of a type @@ -89,6 +92,17 @@ type GraphGateway struct { lk sync.RWMutex notifiers map[Notifier]struct{} + metrics *GraphGatewayMetrics +} + +type GraphGatewayMetrics struct { + carFetchAttemptMetric prometheus.Counter + carBlocksFetchedMetric prometheus.Counter + blockRecoveryAttemptMetric prometheus.Counter + carParamsMetric *prometheus.CounterVec + + // TODO: what to measure here? + //fetchParamsMetric *prometheus.CounterVec } func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts ...GraphGatewayOption) (*GraphGateway, error) { @@ -138,9 +152,66 @@ func NewGraphGatewayBackend(f CarFetcher, blockFetcher exchange.Fetcher, opts .. namesys: ns, bstore: bs, notifiers: make(map[Notifier]struct{}), + metrics: registerGraphGatewayMetrics(), }, nil } +func registerGraphGatewayMetrics() *GraphGatewayMetrics { + + // How many CAR Fetch attempts we had? Need this to calculate % of various graph request types. + // We only count attempts here, because success/failure with/without retries are provided by caboose: + // - ipfs_caboose_fetch_duration_car_success_count + // - ipfs_caboose_fetch_duration_car_failure_count + // - ipfs_caboose_fetch_duration_car_peer_success_count + // - ipfs_caboose_fetch_duration_car_peer_failure_count + carFetchAttemptMetric := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "ipfs", + Subsystem: "gw_graph_backend", + Name: "car_fetch_attempts", + Help: "The number of times a CAR fetch was attempted by IPFSBackend.", + }) + prometheus.MustRegister(carFetchAttemptMetric) + + // How many blocks were read via CARs? + // Need this as a baseline to reason about error ratio vs raw_block_recovery_attempts. + carBlocksFetchedMetric := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "ipfs", + Subsystem: "gw_graph_backend", + Name: "car_blocks_fetched", + Help: "The number of blocks successfully read via CAR fetch.", + }) + prometheus.MustRegister(carBlocksFetchedMetric) + + // How many times CAR response was not enough or just failed, and we had to read a block via ?format=raw + // We only count attempts here, because success/failure with/without retries are provided by caboose: + // - ipfs_caboose_fetch_duration_block_success_count + // - ipfs_caboose_fetch_duration_block_failure_count + // - ipfs_caboose_fetch_duration_block_peer_success_count + // - ipfs_caboose_fetch_duration_block_peer_failure_count + blockRecoveryAttemptMetric := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "ipfs", + Subsystem: "gw_graph_backend", + Name: "raw_block_recovery_attempts", + Help: "The number of ?format=raw block fetch attempts due to GraphGateway failure (CAR fetch error, missing block in CAR response, or a block evicted from cache too soon).", + }) + prometheus.MustRegister(blockRecoveryAttemptMetric) + + carParamsMetric := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ipfs", + Subsystem: "gw_graph_backend", + Name: "car_fetch_params", + Help: "How many times specific CAR parameter was used during CAR data fetch.", + }, []string{"depth", "bytes"}) + prometheus.MustRegister(carParamsMetric) + + return &GraphGatewayMetrics{ + carFetchAttemptMetric, + carBlocksFetchedMetric, + blockRecoveryAttemptMetric, + carParamsMetric, + } +} + /* Implementation iteration plan: @@ -159,18 +230,21 @@ func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx con startingExchange: carFetchingExch, followupExchange: &blockFetcherExchWrapper{api.blockFetcher}, handoffCh: doneWithFetcher, + metrics: api.metrics, } api.lk.Lock() api.notifiers[exch] = struct{}{} api.lk.Unlock() - go func() { + go func(metrics *GraphGatewayMetrics) { defer func() { if r := recover(); r != nil { - fmt.Println("Recovered fetcher error", r) + // TODO: move to Debugw? + graphLog.Errorw("Recovered fetcher error", "error", r) } }() + metrics.carFetchAttemptMetric.Inc() err := api.fetcher.Fetch(ctx, path, func(resource string, reader io.Reader) error { cr, err := car.NewCarReader(reader) if err != nil { @@ -187,18 +261,21 @@ func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx con if err := bstore.Put(ctx, blk); err != nil { return err } + metrics.carBlocksFetchedMetric.Inc() api.notifyAllOngoingRequests(ctx, blk) } }) if err != nil { - graphLog.Error(err) + // TODO: move to debug? + graphLog.Errorw("car Fetch failed", "error", err) } if err := carFetchingExch.Close(); err != nil { - graphLog.Error(err) + // TODO: move to debug? + graphLog.Errorw("carFetchingExch.Close()", "error", err) } doneWithFetcher <- struct{}{} close(doneWithFetcher) - }() + }(api.metrics) bserv := blockservice.New(bstore, exch) blkgw, err := gateway.NewBlocksGateway(bserv) @@ -218,7 +295,7 @@ func (api *GraphGateway) notifyAllOngoingRequests(ctx context.Context, blks ...b for n := range api.notifiers { err := n.NotifyNewBlocks(ctx, blks...) if err != nil { - graphLog.Error(fmt.Errorf("notifyAllOngoingRequests failed: %w", err)) + graphLog.Errorw("notifyAllOngoingRequests failed", "error", err) } } api.lk.RUnlock() @@ -264,6 +341,7 @@ func wrapNodeWithClose[T files.Node](node T, closeFn func()) (T, error) { } func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "1", "bytes": ""}).Inc() blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") if err != nil { return gateway.ContentPathMetadata{}, nil, err @@ -277,9 +355,34 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath) (g return md, gr, err } -func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePath, getRange ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) { - // TODO: actually implement ranges - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") +func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePath, httpRanges ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) { + rangeCount := len(httpRanges) + api.metrics.carParamsMetric.With(prometheus.Labels{ + "depth": "1", + "bytes": fmt.Sprintf("(request ranges: %d)", rangeCount), // we dont pass specific ranges, we want deterministic number of CounterVec labels + }).Inc() + + carParams := "?format=car&depth=1" + + // TODO: refactor this if we ever merge Get and GetRange + if rangeCount > 0 { + + bytesBuilder := strings.Builder{} + bytesBuilder.WriteString("&bytes=") + for i, r := range httpRanges { + bytesBuilder.WriteString(strconv.FormatUint(r.From, 10)) + bytesBuilder.WriteString("-") + if r.To != nil { + bytesBuilder.WriteString(strconv.FormatInt(*r.To, 10)) + } + if i != rangeCount-1 { + bytesBuilder.WriteString(",") + } + } + carParams += bytesBuilder.String() + } + + blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+carParams) if err != nil { return gateway.ContentPathMetadata{}, nil, err } @@ -295,6 +398,7 @@ func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePat } func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "all", "bytes": ""}).Inc() blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=all") if err != nil { return gateway.ContentPathMetadata{}, nil, err @@ -311,6 +415,8 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) } func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "0", "bytes": ""}).Inc() + // TODO: if path is `/ipfs/cid`, we should use ?format=raw blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") if err != nil { return gateway.ContentPathMetadata{}, nil, err @@ -327,6 +433,10 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat } func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { + api.metrics.carParamsMetric.With(prometheus.Labels{ + "depth": "", + "bytes": "0:1023 (Head)", + }).Inc() blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&bytes=0:1023") if err != nil { return gateway.ContentPathMetadata{}, nil, err @@ -343,6 +453,7 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( } func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "0", "bytes": ""}).Inc() blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") if err != nil { return gateway.ContentPathMetadata{}, err @@ -352,6 +463,7 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable } func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) { + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "", "bytes": ""}).Inc() blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car") if err != nil { return gateway.ContentPathMetadata{}, nil, nil, err @@ -479,6 +591,7 @@ var _ exchange.Interface = (*inboundBlockExchange)(nil) type handoffExchange struct { startingExchange, followupExchange exchange.Interface handoffCh <-chan struct{} + metrics *GraphGatewayMetrics } func (f *handoffExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { @@ -493,7 +606,8 @@ func (f *handoffExchange) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block select { case <-f.handoffCh: - graphLog.Infof("needed to use use a backup fetcher for cid %s", c) + graphLog.Debugw("switching to backup block fetcher", "cid", c) + f.metrics.blockRecoveryAttemptMetric.Inc() return f.followupExchange.GetBlock(ctx, c) case <-ctx.Done(): return nil, ctx.Err() @@ -533,10 +647,11 @@ func (f *handoffExchange) GetBlocks(ctx context.Context, cids []cid.Cid) (<-chan newCidArr = append(newCidArr, c) } } - graphLog.Infof("needed to use use a backup fetcher for cids %v", newCidArr) + graphLog.Debugw("needed to use use a backup fetcher for cids", "cids", newCidArr) + f.metrics.blockRecoveryAttemptMetric.Add(float64(len(newCidArr))) fch, err := f.followupExchange.GetBlocks(ctx, newCidArr) if err != nil { - graphLog.Error(fmt.Errorf("error getting blocks from followup exchange %w", err)) + graphLog.Errorw("error getting blocks from followupExchange", "error", err) return } for cs.Len() < len(cids) { diff --git a/main.go b/main.go index aa19dee..7016f31 100644 --- a/main.go +++ b/main.go @@ -67,6 +67,7 @@ See documentation at: https://github.com/ipfs/bifrost-gateway/#readme`, } log.Printf("Starting %s %s", name, version) + registerVersionMetric(version) cdns := newCachedDNS(dnsCacheRefreshInterval) defer cdns.Close() diff --git a/version.go b/version.go index b9982fb..0bc8946 100644 --- a/version.go +++ b/version.go @@ -3,6 +3,8 @@ package main import ( "runtime/debug" "time" + + "github.com/prometheus/client_golang/prometheus" ) var name = "bifrost-gateway" @@ -37,3 +39,15 @@ func buildVersion() string { } return "dev-build" } + +func registerVersionMetric(version string) { + m := prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "ipfs", + Subsystem: "bifrost_gateway", + Name: "info", + Help: "Information about bifrost-gateway instance.", + ConstLabels: prometheus.Labels{"version": version}, + }) + prometheus.MustRegister(m) + m.Set(1) +} From a1c8bdb963cc83461d5ccd3ae7b8a024860321bd Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 1 Apr 2023 02:17:13 +0200 Subject: [PATCH 26/27] refactor: merge Get and GetRange https://github.com/ipfs/boxo/pull/240#pullrequestreview-1365554977 --- blockstore_proxy.go | 2 +- go.mod | 2 +- go.sum | 4 +-- lib/blockstore_cache.go | 2 +- lib/graph_gateway.go | 63 ++++++++++++---------------------------- test-backend/handlers.go | 29 +++++++++--------- 6 files changed, 39 insertions(+), 63 deletions(-) diff --git a/blockstore_proxy.go b/blockstore_proxy.go index 6b58e96..1a31846 100644 --- a/blockstore_proxy.go +++ b/blockstore_proxy.go @@ -12,7 +12,7 @@ import ( "github.com/ipfs/bifrost-gateway/lib" blockstore "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/go-block-format" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ) diff --git a/go.mod b/go.mod index 770fbec..e05ca1f 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/filecoin-saturn/caboose v0.0.0-20230329203940-7c08345c244d github.com/gogo/protobuf v1.3.2 github.com/hashicorp/golang-lru/v2 v2.0.1 - github.com/ipfs/boxo v0.8.0-rc3.0.20230331021433-e49cc43d95ad + github.com/ipfs/boxo v0.8.0-rc3.0.20230401000028-de9daf5fc542 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.0 github.com/ipfs/go-ipld-format v0.4.0 diff --git a/go.sum b/go.sum index 0e576a6..78a7ea9 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7P github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.0-rc3.0.20230331021433-e49cc43d95ad h1:olq3866OLC+aCW3hImmUtze0Nh8lbFfciTSeq09xfQU= -github.com/ipfs/boxo v0.8.0-rc3.0.20230331021433-e49cc43d95ad/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= +github.com/ipfs/boxo v0.8.0-rc3.0.20230401000028-de9daf5fc542 h1:2aYk8I/b1NvD2AzsNMhFrCbPjzuarcHVBdGtl0KRST4= +github.com/ipfs/boxo v0.8.0-rc3.0.20230401000028-de9daf5fc542/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/lib/blockstore_cache.go b/lib/blockstore_cache.go index 7e986a6..a41778c 100644 --- a/lib/blockstore_cache.go +++ b/lib/blockstore_cache.go @@ -19,7 +19,7 @@ import ( const DefaultCacheBlockStoreSize = 1024 -var cacheLog = golog.Logger("cache-blockstore") +var cacheLog = golog.Logger("cache/block") func NewCacheBlockStore(size int) (blockstore.Blockstore, error) { c, err := lru.New2Q[string, []byte](size) diff --git a/lib/graph_gateway.go b/lib/graph_gateway.go index 0df8059..0d6a73e 100644 --- a/lib/graph_gateway.go +++ b/lib/graph_gateway.go @@ -36,7 +36,7 @@ import ( "go.uber.org/multierr" ) -var graphLog = golog.Logger("graph-gateway") +var graphLog = golog.Logger("backend/graph") // type DataCallback = func(resource string, reader io.Reader) error // TODO: Don't use a caboose type, perhaps ask them to use a type alias instead of a type @@ -201,7 +201,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics { Subsystem: "gw_graph_backend", Name: "car_fetch_params", Help: "How many times specific CAR parameter was used during CAR data fetch.", - }, []string{"depth", "bytes"}) + }, []string{"depth", "ranges"}) // we use 'ranges' instead of 'bytes' here because we only caount the number of ranges present prometheus.MustRegister(carParamsMetric) return &GraphGatewayMetrics{ @@ -241,7 +241,7 @@ func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx con defer func() { if r := recover(); r != nil { // TODO: move to Debugw? - graphLog.Errorw("Recovered fetcher error", "error", r) + graphLog.Errorw("Recovered fetcher error", "path", path, "error", r) } }() metrics.carFetchAttemptMetric.Inc() @@ -266,11 +266,9 @@ func (api *GraphGateway) loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx con } }) if err != nil { - // TODO: move to debug? - graphLog.Errorw("car Fetch failed", "error", err) + graphLog.Debugw("car Fetch failed", "path", path, "error", err) } if err := carFetchingExch.Close(); err != nil { - // TODO: move to debug? graphLog.Errorw("carFetchingExch.Close()", "error", err) } doneWithFetcher <- struct{}{} @@ -340,36 +338,17 @@ func wrapNodeWithClose[T files.Node](node T, closeFn func()) (T, error) { } } -func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "1", "bytes": ""}).Inc() - blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=1") - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - md, gr, err := blkgw.Get(ctx, path) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - //TODO: interfaces here aren't good enough so we're getting around the problem this way - runtime.SetFinalizer(gr, func(_ *gateway.GetResponse) { closeFn() }) - return md, gr, err -} - -func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePath, httpRanges ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) { - rangeCount := len(httpRanges) - api.metrics.carParamsMetric.With(prometheus.Labels{ - "depth": "1", - "bytes": fmt.Sprintf("(request ranges: %d)", rangeCount), // we dont pass specific ranges, we want deterministic number of CounterVec labels - }).Inc() +func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, byteRanges ...gateway.ByteRange) (gateway.ContentPathMetadata, *gateway.GetResponse, error) { + rangeCount := len(byteRanges) + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "1", "ranges": strconv.Itoa(rangeCount)}).Inc() carParams := "?format=car&depth=1" - // TODO: refactor this if we ever merge Get and GetRange + // fetch CAR with &bytes= to get minimal set of blocks for the request if rangeCount > 0 { - bytesBuilder := strings.Builder{} bytesBuilder.WriteString("&bytes=") - for i, r := range httpRanges { + for i, r := range byteRanges { bytesBuilder.WriteString(strconv.FormatUint(r.From, 10)) bytesBuilder.WriteString("-") if r.To != nil { @@ -386,19 +365,18 @@ func (api *GraphGateway) GetRange(ctx context.Context, path gateway.ImmutablePat if err != nil { return gateway.ContentPathMetadata{}, nil, err } - md, f, err := blkgw.GetRange(ctx, path) - if err != nil { - return gateway.ContentPathMetadata{}, nil, err - } - f, err = wrapNodeWithClose(f, closeFn) + md, gr, err := blkgw.Get(ctx, path, byteRanges...) if err != nil { return gateway.ContentPathMetadata{}, nil, err } - return md, f, nil + + //TODO: interfaces here aren't good enough so we're getting around the problem this way + runtime.SetFinalizer(gr, func(_ *gateway.GetResponse) { closeFn() }) + return md, gr, nil } func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "all", "bytes": ""}).Inc() + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "all", "ranges": "0"}).Inc() blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=all") if err != nil { return gateway.ContentPathMetadata{}, nil, err @@ -415,7 +393,7 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) } func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "0", "bytes": ""}).Inc() + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "0", "ranges": "0"}).Inc() // TODO: if path is `/ipfs/cid`, we should use ?format=raw blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") if err != nil { @@ -433,10 +411,7 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat } func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{ - "depth": "", - "bytes": "0:1023 (Head)", - }).Inc() + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "", "ranges": "1"}).Inc() blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&bytes=0:1023") if err != nil { return gateway.ContentPathMetadata{}, nil, err @@ -453,7 +428,7 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) ( } func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "0", "bytes": ""}).Inc() + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "0", "ranges": "0"}).Inc() blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&depth=0") if err != nil { return gateway.ContentPathMetadata{}, err @@ -463,7 +438,7 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable } func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) { - api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "", "bytes": ""}).Inc() + api.metrics.carParamsMetric.With(prometheus.Labels{"depth": "", "ranges": "0"}).Inc() blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car") if err != nil { return gateway.ContentPathMetadata{}, nil, nil, err diff --git a/test-backend/handlers.go b/test-backend/handlers.go index 23c5d85..ad81d40 100644 --- a/test-backend/handlers.go +++ b/test-backend/handlers.go @@ -4,6 +4,15 @@ import ( "bytes" "context" "fmt" + "io" + "net/http" + "net/url" + "runtime/debug" + "strconv" + "strings" + "sync" + "time" + "github.com/ipfs/boxo/fetcher" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway" @@ -13,7 +22,7 @@ import ( unixfile "github.com/ipfs/boxo/ipld/unixfs/file" "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/path/resolver" - "github.com/ipfs/go-block-format" + blocks "github.com/ipfs/go-block-format" format "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-unixfsnode" dagpb "github.com/ipld/go-codec-dagpb" @@ -23,14 +32,6 @@ import ( "github.com/ipld/go-ipld-prime/schema" "github.com/ipld/go-ipld-prime/traversal" "github.com/ipld/go-ipld-prime/traversal/selector" - "io" - "net/http" - "net/url" - "runtime/debug" - "strconv" - "strings" - "sync" - "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -189,9 +190,9 @@ func simpleSelectorToCar(ctx context.Context, bsrv blockservice.BlockService, p if hasDepth && !(depthStr == "0" || depthStr == "1" || depthStr == "all") { return nil, fmt.Errorf("depth type: %s not supported", depthStr) } - var getRange *gateway.GetRange + var getRange *gateway.ByteRange if hasRange { - getRange, err = rangeStrToGetRange(rangeStr) + getRange, err = rangeStrToByteRange(rangeStr) if err != nil { return nil, err } @@ -478,7 +479,7 @@ func blockOpener(ctx context.Context, ng format.NodeGetter) ipld.BlockReadOpener var _ fetcher.Fetcher = (*nodeGetterFetcherSingleUseFactory)(nil) var _ fetcher.Factory = (*nodeGetterFetcherSingleUseFactory)(nil) -func rangeStrToGetRange(rangeStr string) (*gateway.GetRange, error) { +func rangeStrToByteRange(rangeStr string) (*gateway.ByteRange, error) { rangeElems := strings.Split(rangeStr, ":") if len(rangeElems) > 2 { return nil, fmt.Errorf("invalid range") @@ -489,7 +490,7 @@ func rangeStrToGetRange(rangeStr string) (*gateway.GetRange, error) { } if rangeElems[1] == "*" { - return &gateway.GetRange{ + return &gateway.ByteRange{ From: first, To: nil, }, nil @@ -509,7 +510,7 @@ func rangeStrToGetRange(rangeStr string) (*gateway.GetRange, error) { return nil, fmt.Errorf("invalid range") } - return &gateway.GetRange{ + return &gateway.ByteRange{ From: first, To: &second, }, nil From 8c3386f4a4a004ac6ca6a4e72c67fff5504e2ea2 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 3 Apr 2023 19:32:09 +0200 Subject: [PATCH 27/27] chore: boxo v0.8.0-rc4 This is the updated tag after undesired binary got merged. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e05ca1f..050c368 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/filecoin-saturn/caboose v0.0.0-20230329203940-7c08345c244d github.com/gogo/protobuf v1.3.2 github.com/hashicorp/golang-lru/v2 v2.0.1 - github.com/ipfs/boxo v0.8.0-rc3.0.20230401000028-de9daf5fc542 + github.com/ipfs/boxo v0.8.0-rc4 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.0 github.com/ipfs/go-ipld-format v0.4.0 diff --git a/go.sum b/go.sum index 78a7ea9..a35f9b8 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,8 @@ github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7P github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.0-rc3.0.20230401000028-de9daf5fc542 h1:2aYk8I/b1NvD2AzsNMhFrCbPjzuarcHVBdGtl0KRST4= -github.com/ipfs/boxo v0.8.0-rc3.0.20230401000028-de9daf5fc542/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= +github.com/ipfs/boxo v0.8.0-rc4 h1:TT9uY/cw6yjmeKjfJPKaGZIYSJmV6B9Bd/O5Rem6MfA= +github.com/ipfs/boxo v0.8.0-rc4/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=