From 7e03f4c7915100685834e50f0d0e89ba43325f5b Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 31 Jan 2023 12:45:41 +0100 Subject: [PATCH] feat: feat: use new gateway api from go-libipfs, deprecate writable gateway --- cmd/ipfs/daemon.go | 8 +- config/gateway.go | 6 +- config/init.go | 1 - core/corehttp/gateway.go | 87 +++++++- core/corehttp/gateway_writable.go | 265 +++++++++++++++++++++++++ docs/config.md | 2 +- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 +- go.mod | 2 +- go.sum | 4 +- 10 files changed, 363 insertions(+), 18 deletions(-) create mode 100644 core/corehttp/gateway_writable.go diff --git a/cmd/ipfs/daemon.go b/cmd/ipfs/daemon.go index 12b3f4d9cccc..98922b601036 100644 --- a/cmd/ipfs/daemon.go +++ b/cmd/ipfs/daemon.go @@ -162,7 +162,7 @@ Headers. cmds.StringOption(initProfileOptionKwd, "Configuration profiles to apply for --init. See ipfs init --help for more"), cmds.StringOption(routingOptionKwd, "Overrides the routing option").WithDefault(routingOptionDefaultKwd), cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem using FUSE (experimental)"), - cmds.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE)"), + cmds.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE, DEPRECATED)"), cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount). Defaults to config setting."), cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount). Defaults to config setting."), cmds.BoolOption(unrestrictedAPIAccessKwd, "Allow API access to unlisted hashes"), @@ -791,7 +791,11 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e writable, writableOptionFound := req.Options[writableKwd].(bool) if !writableOptionFound { - writable = cfg.Gateway.Writable + writable = cfg.Gateway.Writable.WithDefault(false) + } + + if writable { + log.Error("serveHTTPGateway: writable gateways are DEPRECATED and will be removed in future versions") } listeners, err := sockets.TakeListeners("io.ipfs.gateway") diff --git a/config/gateway.go b/config/gateway.go index ad01b263b366..a4c3ea5e4093 100644 --- a/config/gateway.go +++ b/config/gateway.go @@ -38,9 +38,9 @@ type Gateway struct { // should be redirected. RootRedirect string - // Writable enables PUT/POST request handling by this gateway. Usually, - // writing is done through the API, not the gateway. - Writable bool + // DEPRECATED: Writable enables PUT/POST request handling by this gateway. + // Usually, writing is done through the API, not the gateway. + Writable Flag `json:",omitempty"` // PathPrefixes was removed: https://github.com/ipfs/go-ipfs/issues/7702 PathPrefixes []string diff --git a/config/init.go b/config/init.go index 621ff95f3531..288f8a1d58c2 100644 --- a/config/init.go +++ b/config/init.go @@ -65,7 +65,6 @@ func InitWithIdentity(identity Identity) (*Config, error) { Gateway: Gateway{ RootRedirect: "", - Writable: false, NoFetch: false, PathPrefixes: []string{}, HTTPHeaders: map[string][]string{ diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go index d5eccf73c275..e5d91f385f11 100644 --- a/core/corehttp/gateway.go +++ b/core/corehttp/gateway.go @@ -1,12 +1,19 @@ package corehttp import ( + "context" "fmt" + "io" "net" "net/http" + cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/files" "github.com/ipfs/go-libipfs/gateway" + iface "github.com/ipfs/interface-go-ipfs-core" options "github.com/ipfs/interface-go-ipfs-core/options" + "github.com/ipfs/interface-go-ipfs-core/path" version "github.com/ipfs/kubo" core "github.com/ipfs/kubo/core" coreapi "github.com/ipfs/kubo/core/coreapi" @@ -38,15 +45,45 @@ func GatewayOption(writable bool, paths ...string) ServeOption { return nil, err } - gateway := gateway.NewHandler(gateway.Config{ - Headers: headers, - Writable: writable, - }, api, offlineAPI) + gatewayConfig := gateway.Config{ + Headers: headers, + } + + gatewayAPI := &gatewayAPI{ + api: api, + offlineAPI: offlineAPI, + } + gateway := gateway.NewHandler(gatewayConfig, gatewayAPI) gateway = otelhttp.NewHandler(gateway, "Gateway.Request") + var writableGateway *writableGatewayHandler + if writable { + writableGateway = &writableGatewayHandler{ + config: &gatewayConfig, + api: api, + } + } + for _, p := range paths { - mux.Handle(p+"/", gateway) + mux.HandleFunc(p+"/", func(w http.ResponseWriter, r *http.Request) { + if writable { + switch r.Method { + case http.MethodPost: + writableGateway.postHandler(w, r) + case http.MethodDelete: + writableGateway.deleteHandler(w, r) + case http.MethodPut: + writableGateway.putHandler(w, r) + default: + gateway.ServeHTTP(w, r) + } + + return + } + + gateway.ServeHTTP(w, r) + }) } return mux, nil } @@ -62,3 +99,43 @@ func VersionOption() ServeOption { return mux, nil } } + +type gatewayAPI struct { + api iface.CoreAPI + offlineAPI iface.CoreAPI +} + +func (gw *gatewayAPI) GetUnixFsNode(ctx context.Context, pth path.Path) (files.Node, error) { + return gw.api.Unixfs().Get(ctx, pth) +} + +func (gw *gatewayAPI) LsUnixFsDir(ctx context.Context, pth path.Path, opts ...options.UnixfsLsOption) (<-chan iface.DirEntry, error) { + return gw.api.Unixfs().Ls(ctx, pth, opts...) +} + +func (gw *gatewayAPI) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) { + r, err := gw.api.Block().Get(ctx, path.IpfsPath(cid)) + if err != nil { + return nil, err + } + + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + return blocks.NewBlockWithCid(data, cid) +} + +func (gw *gatewayAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { + return gw.api.Routing().Get(ctx, "/ipns/"+c.String()) +} + +func (gw *gatewayAPI) IsCached(ctx context.Context, pth path.Path) bool { + _, err := gw.offlineAPI.Block().Stat(ctx, pth) + return err == nil +} + +func (gw *gatewayAPI) ResolvePath(ctx context.Context, pth path.Path) (path.Resolved, error) { + return gw.api.ResolvePath(ctx, pth) +} diff --git a/core/corehttp/gateway_writable.go b/core/corehttp/gateway_writable.go new file mode 100644 index 000000000000..89a2973ac23b --- /dev/null +++ b/core/corehttp/gateway_writable.go @@ -0,0 +1,265 @@ +package corehttp + +import ( + "context" + "fmt" + "net/http" + "os" + gopath "path" + + cid "github.com/ipfs/go-cid" + ipld "github.com/ipfs/go-ipld-format" + "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/go-libipfs/gateway" + dag "github.com/ipfs/go-merkledag" + "github.com/ipfs/go-mfs" + path "github.com/ipfs/go-path" + "github.com/ipfs/go-path/resolver" + iface "github.com/ipfs/interface-go-ipfs-core" + routing "github.com/libp2p/go-libp2p/core/routing" +) + +const ( + ipfsPathPrefix = "/ipfs/" +) + +type writableGatewayHandler struct { + api iface.CoreAPI + config *gateway.Config +} + +func (i *writableGatewayHandler) addUserHeaders(w http.ResponseWriter) { + for k, v := range i.config.Headers { + w.Header()[k] = v + } +} + +func (i *writableGatewayHandler) postHandler(w http.ResponseWriter, r *http.Request) { + p, err := i.api.Unixfs().Add(r.Context(), files.NewReaderFile(r.Body)) + if err != nil { + internalWebError(w, err) + return + } + + i.addUserHeaders(w) // ok, _now_ write user's headers. + w.Header().Set("IPFS-Hash", p.Cid().String()) + log.Debugw("CID created, http redirect", "from", r.URL, "to", p, "status", http.StatusCreated) + http.Redirect(w, r, p.String(), http.StatusCreated) +} + +func (i *writableGatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + ds := i.api.Dag() + + // Parse the path + rootCid, newPath, err := parseIpfsPath(r.URL.Path) + if err != nil { + webError(w, "WritableGateway: failed to parse the path", err, http.StatusBadRequest) + return + } + if newPath == "" || newPath == "/" { + http.Error(w, "WritableGateway: empty path", http.StatusBadRequest) + return + } + newDirectory, newFileName := gopath.Split(newPath) + + // Resolve the old root. + + rnode, err := ds.Get(ctx, rootCid) + if err != nil { + webError(w, "WritableGateway: Could not create DAG from request", err, http.StatusInternalServerError) + return + } + + pbnd, ok := rnode.(*dag.ProtoNode) + if !ok { + webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest) + return + } + + // Create the new file. + newFilePath, err := i.api.Unixfs().Add(ctx, files.NewReaderFile(r.Body)) + if err != nil { + webError(w, "WritableGateway: could not create DAG from request", err, http.StatusInternalServerError) + return + } + + newFile, err := ds.Get(ctx, newFilePath.Cid()) + if err != nil { + webError(w, "WritableGateway: failed to resolve new file", err, http.StatusInternalServerError) + return + } + + // Patch the new file into the old root. + + root, err := mfs.NewRoot(ctx, ds, pbnd, nil) + if err != nil { + webError(w, "WritableGateway: failed to create MFS root", err, http.StatusBadRequest) + return + } + + if newDirectory != "" { + err := mfs.Mkdir(root, newDirectory, mfs.MkdirOpts{Mkparents: true, Flush: false}) + if err != nil { + webError(w, "WritableGateway: failed to create MFS directory", err, http.StatusInternalServerError) + return + } + } + dirNode, err := mfs.Lookup(root, newDirectory) + if err != nil { + webError(w, "WritableGateway: failed to lookup directory", err, http.StatusInternalServerError) + return + } + dir, ok := dirNode.(*mfs.Directory) + if !ok { + http.Error(w, "WritableGateway: target directory is not a directory", http.StatusBadRequest) + return + } + err = dir.Unlink(newFileName) + switch err { + case os.ErrNotExist, nil: + default: + webError(w, "WritableGateway: failed to replace existing file", err, http.StatusBadRequest) + return + } + err = dir.AddChild(newFileName, newFile) + if err != nil { + webError(w, "WritableGateway: failed to link file into directory", err, http.StatusInternalServerError) + return + } + nnode, err := root.GetDirectory().GetNode() + if err != nil { + webError(w, "WritableGateway: failed to finalize", err, http.StatusInternalServerError) + return + } + newcid := nnode.Cid() + + i.addUserHeaders(w) // ok, _now_ write user's headers. + w.Header().Set("IPFS-Hash", newcid.String()) + + redirectURL := gopath.Join(ipfsPathPrefix, newcid.String(), newPath) + log.Debugw("CID replaced, redirect", "from", r.URL, "to", redirectURL, "status", http.StatusCreated) + http.Redirect(w, r, redirectURL, http.StatusCreated) +} + +func (i *writableGatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // parse the path + + rootCid, newPath, err := parseIpfsPath(r.URL.Path) + if err != nil { + webError(w, "WritableGateway: failed to parse the path", err, http.StatusBadRequest) + return + } + if newPath == "" || newPath == "/" { + http.Error(w, "WritableGateway: empty path", http.StatusBadRequest) + return + } + directory, filename := gopath.Split(newPath) + + // lookup the root + + rootNodeIPLD, err := i.api.Dag().Get(ctx, rootCid) + if err != nil { + webError(w, "WritableGateway: failed to resolve root CID", err, http.StatusInternalServerError) + return + } + rootNode, ok := rootNodeIPLD.(*dag.ProtoNode) + if !ok { + http.Error(w, "WritableGateway: empty path", http.StatusInternalServerError) + return + } + + // construct the mfs root + + root, err := mfs.NewRoot(ctx, i.api.Dag(), rootNode, nil) + if err != nil { + webError(w, "WritableGateway: failed to construct the MFS root", err, http.StatusBadRequest) + return + } + + // lookup the parent directory + + parentNode, err := mfs.Lookup(root, directory) + if err != nil { + webError(w, "WritableGateway: failed to look up parent", err, http.StatusInternalServerError) + return + } + + parent, ok := parentNode.(*mfs.Directory) + if !ok { + http.Error(w, "WritableGateway: parent is not a directory", http.StatusInternalServerError) + return + } + + // delete the file + + switch parent.Unlink(filename) { + case nil, os.ErrNotExist: + default: + webError(w, "WritableGateway: failed to remove file", err, http.StatusInternalServerError) + return + } + + nnode, err := root.GetDirectory().GetNode() + if err != nil { + webError(w, "WritableGateway: failed to finalize", err, http.StatusInternalServerError) + return + } + ncid := nnode.Cid() + + i.addUserHeaders(w) // ok, _now_ write user's headers. + w.Header().Set("IPFS-Hash", ncid.String()) + + redirectURL := gopath.Join(ipfsPathPrefix+ncid.String(), directory) + // note: StatusCreated is technically correct here as we created a new resource. + log.Debugw("CID deleted, redirect", "from", r.RequestURI, "to", redirectURL, "status", http.StatusCreated) + http.Redirect(w, r, redirectURL, http.StatusCreated) +} + +func parseIpfsPath(p string) (cid.Cid, string, error) { + rootPath, err := path.ParsePath(p) + if err != nil { + return cid.Cid{}, "", err + } + + // Check the path. + rsegs := rootPath.Segments() + if rsegs[0] != "ipfs" { + return cid.Cid{}, "", fmt.Errorf("WritableGateway: only ipfs paths supported") + } + + rootCid, err := cid.Decode(rsegs[1]) + if err != nil { + return cid.Cid{}, "", err + } + + return rootCid, path.Join(rsegs[2:]), nil +} + +func webError(w http.ResponseWriter, message string, err error, defaultCode int) { + if _, ok := err.(resolver.ErrNoLink); ok { + webErrorWithCode(w, message, err, http.StatusNotFound) + } else if err == routing.ErrNotFound { + webErrorWithCode(w, message, err, http.StatusNotFound) + } else if ipld.IsNotFound(err) { + webErrorWithCode(w, message, err, http.StatusNotFound) + } else if err == context.DeadlineExceeded { + webErrorWithCode(w, message, err, http.StatusRequestTimeout) + } else { + webErrorWithCode(w, message, err, defaultCode) + } +} + +func webErrorWithCode(w http.ResponseWriter, message string, err error, code int) { + http.Error(w, fmt.Sprintf("%s: %s", message, err), code) + if code >= 500 { + log.Warnf("server error: %s: %s", message, err) + } +} + +// return a 500 error and log +func internalWebError(w http.ResponseWriter, err error) { + webErrorWithCode(w, "internalWebError", err, http.StatusInternalServerError) +} diff --git a/docs/config.md b/docs/config.md index 995872c4f37f..5a84dfbcedad 100644 --- a/docs/config.md +++ b/docs/config.md @@ -682,7 +682,7 @@ Type: `string` (url) ### `Gateway.Writable` -A boolean to configure whether the gateway is writeable or not. +**DEPRECATED**: A boolean to configure whether the gateway is writeable or not. Default: `false` diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index a74bfbf3e03d..7234ace387d0 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496 + github.com/ipfs/go-libipfs v0.4.1-0.20230201082329-1e462b639ca3 github.com/ipfs/interface-go-ipfs-core v0.10.0 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.24.2 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index eb37654f786e..f316ef32d6b4 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -548,8 +548,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= -github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496 h1:RVI31GQCFODREpasIFyVFkS6PjJT2bMwr/Bgr9Ryql4= -github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496/go.mod h1:AAPvZADZ80i+QhGCWNWCsx8IGY0t9C+IBEngLeYtySY= +github.com/ipfs/go-libipfs v0.4.1-0.20230201082329-1e462b639ca3 h1:q+r9JG9T1jYNaUhH18I4aP0DJkU3N65EdYUNqFp++9s= +github.com/ipfs/go-libipfs v0.4.1-0.20230201082329-1e462b639ca3/go.mod h1:S5wg08D/FkeYxeMf8adgt6Mi6ttbA7kSFcQYlmeGHMU= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= diff --git a/go.mod b/go.mod index 5edf9f68a467..94dbe3359085 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/ipfs/go-ipld-git v0.1.1 github.com/ipfs/go-ipld-legacy v0.1.1 github.com/ipfs/go-ipns v0.3.0 - github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496 + github.com/ipfs/go-libipfs v0.4.1-0.20230201082329-1e462b639ca3 github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.9.0 diff --git a/go.sum b/go.sum index fefe9bf71606..b0211e48ecee 100644 --- a/go.sum +++ b/go.sum @@ -570,8 +570,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= -github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496 h1:RVI31GQCFODREpasIFyVFkS6PjJT2bMwr/Bgr9Ryql4= -github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496/go.mod h1:AAPvZADZ80i+QhGCWNWCsx8IGY0t9C+IBEngLeYtySY= +github.com/ipfs/go-libipfs v0.4.1-0.20230201082329-1e462b639ca3 h1:q+r9JG9T1jYNaUhH18I4aP0DJkU3N65EdYUNqFp++9s= +github.com/ipfs/go-libipfs v0.4.1-0.20230201082329-1e462b639ca3/go.mod h1:S5wg08D/FkeYxeMf8adgt6Mi6ttbA7kSFcQYlmeGHMU= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=