From 91a591170f838144e954125483701dab8a154836 Mon Sep 17 00:00:00 2001 From: gmgigi96 Date: Mon, 3 Apr 2023 15:07:56 +0200 Subject: [PATCH 1/4] allow initiate file download for a revision file --- .../grpc/services/gateway/storageprovider.go | 23 ++++-- .../http/services/datagateway/datagateway.go | 15 +++- pkg/rhttp/datatx/utils/download/download.go | 70 ++++++++++++++----- 3 files changed, 84 insertions(+), 24 deletions(-) diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index 68900c5ee5..a4ec8c4bb2 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -52,10 +52,11 @@ import ( // transferClaims are custom claims for a JWT token to be used between the metadata and data gateways. type transferClaims struct { jwt.StandardClaims - Target string `json:"target"` + Target string `json:"target"` + VersionKey string `json:"version_key,omitempty"` } -func (s *svc) sign(_ context.Context, target string) (string, error) { +func (s *svc) sign(_ context.Context, target, versionKey string) (string, error) { // Tus sends a separate request to the datagateway service for every chunk. // For large files, this can take a long time, so we extend the expiration ttl := time.Duration(s.c.TransferExpires) * time.Second @@ -65,7 +66,8 @@ func (s *svc) sign(_ context.Context, target string) (string, error) { Audience: "reva", IssuedAt: time.Now().Unix(), }, - Target: target, + Target: target, + VersionKey: versionKey, } t := jwt.NewWithClaims(jwt.GetSigningMethod("HS256"), claims) @@ -470,6 +472,17 @@ func (s *svc) InitiateFileDownload(ctx context.Context, req *provider.InitiateFi panic("gateway: download: unknown path:" + p) } +func versionKey(req *provider.InitiateFileDownloadRequest) string { + if req.Opaque == nil || req.Opaque.Map == nil { + return "" + } + val := req.Opaque.Map["version_key"] + if val == nil { + return "" + } + return string(val.Value) +} + func (s *svc) initiateFileDownload(ctx context.Context, req *provider.InitiateFileDownloadRequest) (*gateway.InitiateFileDownloadResponse, error) { // TODO(ishank011): enable downloading references spread across storage providers, eg. /eos c, err := s.find(ctx, req.Ref) @@ -506,7 +519,7 @@ func (s *svc) initiateFileDownload(ctx context.Context, req *provider.InitiateFi // TODO(labkode): calculate signature of the whole request? we only sign the URI now. Maybe worth https://tools.ietf.org/html/draft-cavage-http-signatures-11 target := u.String() - token, err := s.sign(ctx, target) + token, err := s.sign(ctx, target, versionKey(req)) if err != nil { return &gateway.InitiateFileDownloadResponse{ Status: status.NewInternal(ctx, err, "error creating signature for download"), @@ -712,7 +725,7 @@ func (s *svc) initiateFileUpload(ctx context.Context, req *provider.InitiateFile // TODO(labkode): calculate signature of the whole request? we only sign the URI now. Maybe worth https://tools.ietf.org/html/draft-cavage-http-signatures-11 target := u.String() - token, err := s.sign(ctx, target) + token, err := s.sign(ctx, target, "") if err != nil { return &gateway.InitiateFileUploadResponse{ Status: status.NewInternal(ctx, err, "error creating signature for upload"), diff --git a/internal/http/services/datagateway/datagateway.go b/internal/http/services/datagateway/datagateway.go index 0c2a460793..57a5a6134a 100644 --- a/internal/http/services/datagateway/datagateway.go +++ b/internal/http/services/datagateway/datagateway.go @@ -52,7 +52,8 @@ func init() { // transferClaims are custom claims for a JWT token to be used between the metadata and data gateways. type transferClaims struct { jwt.StandardClaims - Target string `json:"target"` + Target string `json:"target"` + VersionKey string `json:"version_key,omitempty"` } type config struct { Prefix string `mapstructure:"prefix"` @@ -191,6 +192,12 @@ func (s *svc) doHead(w http.ResponseWriter, r *http.Request) { } httpReq.Header = r.Header + if claims.VersionKey != "" { + q := httpReq.URL.Query() + q.Add("version_key", claims.VersionKey) + httpReq.URL.RawQuery = q.Encode() + } + httpRes, err := httpClient.Do(httpReq) if err != nil { log.Error().Err(err).Msg("error doing HEAD request to data service") @@ -237,6 +244,12 @@ func (s *svc) doGet(w http.ResponseWriter, r *http.Request) { } httpReq.Header = r.Header + if claims.VersionKey != "" { + q := httpReq.URL.Query() + q.Add("version_key", claims.VersionKey) + httpReq.URL.RawQuery = q.Encode() + } + httpRes, err := httpClient.Do(httpReq) if err != nil { log.Error().Err(err).Msg("error doing GET request to data service") diff --git a/pkg/rhttp/datatx/utils/download/download.go b/pkg/rhttp/datatx/utils/download/download.go index 95bb3b3ea5..669df72959 100644 --- a/pkg/rhttp/datatx/utils/download/download.go +++ b/pkg/rhttp/datatx/utils/download/download.go @@ -20,6 +20,7 @@ package download import ( + "context" "fmt" "io" "mime/multipart" @@ -68,30 +69,57 @@ func GetOrHeadFile(w http.ResponseWriter, r *http.Request, fs storage.FS, spaceI } // TODO check preconditions like If-Range, If-Match ... - var md *provider.ResourceInfo - var err error - - // do a stat to set a Content-Length header + var ( + md *provider.ResourceInfo + content io.ReadCloser + size int64 + err error + ) + // do a stat to get the mime type if md, err = fs.GetMD(ctx, ref, nil); err != nil { handleError(w, &sublog, err, "stat") return } + mimeType := md.MimeType + + if versionKey := r.URL.Query().Get("version_key"); versionKey != "" { + // the request is for a version file + stat, err := statRevision(ctx, fs, ref, versionKey) + if err != nil { + handleError(w, &sublog, err, "stat revision") + return + } + size = int64(stat.Size) + content, err = fs.DownloadRevision(ctx, ref, versionKey) + if err != nil { + handleError(w, &sublog, err, "download revision") + return + } + } else { + size = int64(md.Size) + content, err = fs.Download(ctx, ref) + if err != nil { + handleError(w, &sublog, err, "download") + return + } + } + defer content.Close() var ranges []HTTPRange if r.Header.Get("Range") != "" { - ranges, err = ParseRange(r.Header.Get("Range"), int64(md.Size)) + ranges, err = ParseRange(r.Header.Get("Range"), size) if err != nil { if err == ErrNoOverlap { - w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", md.Size)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", size)) } sublog.Error().Err(err).Interface("md", md).Interface("ranges", ranges).Msg("range request not satisfiable") w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) return } - if SumRangesSize(ranges) > int64(md.Size) { + if SumRangesSize(ranges) > size { // The total number of bytes in all the ranges // is larger than the size of the file by // itself, so this is probably an attack, or a @@ -100,15 +128,8 @@ func GetOrHeadFile(w http.ResponseWriter, r *http.Request, fs storage.FS, spaceI } } - content, err := fs.Download(ctx, ref) - if err != nil { - handleError(w, &sublog, err, "download") - return - } - defer content.Close() - code := http.StatusOK - sendSize := int64(md.Size) + sendSize := size var sendContent io.Reader = content var s io.Seeker @@ -146,9 +167,9 @@ func GetOrHeadFile(w http.ResponseWriter, r *http.Request, fs storage.FS, spaceI } sendSize = ra.Length code = http.StatusPartialContent - w.Header().Set("Content-Range", ra.ContentRange(int64(md.Size))) + w.Header().Set("Content-Range", ra.ContentRange(size)) case len(ranges) > 1: - sendSize = RangesMIMESize(ranges, md.MimeType, int64(md.Size)) + sendSize = RangesMIMESize(ranges, mimeType, size) code = http.StatusPartialContent pr, pw := io.Pipe() @@ -158,7 +179,7 @@ func GetOrHeadFile(w http.ResponseWriter, r *http.Request, fs storage.FS, spaceI defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. go func() { for _, ra := range ranges { - part, err := mw.CreatePart(ra.MimeHeader(md.MimeType, int64(md.Size))) + part, err := mw.CreatePart(ra.MimeHeader(mimeType, size)) if err != nil { _ = pw.CloseWithError(err) // CloseWithError always returns nil return @@ -197,6 +218,19 @@ func GetOrHeadFile(w http.ResponseWriter, r *http.Request, fs storage.FS, spaceI } } +func statRevision(ctx context.Context, fs storage.FS, ref *provider.Reference, revisionKey string) (*provider.FileVersion, error) { + versions, err := fs.ListRevisions(ctx, ref) + if err != nil { + return nil, err + } + for _, v := range versions { + if v.Key == revisionKey { + return v, nil + } + } + return nil, errtypes.NotFound("version not found") +} + func handleError(w http.ResponseWriter, log *zerolog.Logger, err error, action string) { switch err.(type) { case errtypes.IsNotFound: From 17afecd90596c0cb1392749ac45283e181be0ec8 Mon Sep 17 00:00:00 2001 From: gmgigi96 Date: Mon, 3 Apr 2023 15:19:27 +0200 Subject: [PATCH 2/4] download version file from webdav --- .../http/services/owncloud/ocdav/versions.go | 49 +++++++++++++++++++ pkg/storage/utils/downloader/downloader.go | 20 ++++++-- .../utils/downloader/mock/downloader_mock.go | 2 +- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/internal/http/services/owncloud/ocdav/versions.go b/internal/http/services/owncloud/ocdav/versions.go index 6b98f8b998..54ce347c0e 100644 --- a/internal/http/services/owncloud/ocdav/versions.go +++ b/internal/http/services/owncloud/ocdav/versions.go @@ -20,14 +20,17 @@ package ocdav import ( "context" + "fmt" "net/http" "path" + "path/filepath" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/router" + "github.com/cs3org/reva/pkg/storage/utils/downloader" rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils/resourceid" ) @@ -74,6 +77,10 @@ func (h *VersionsHandler) Handler(s *svc, rid *provider.ResourceId) http.Handler h.doRestore(w, r, s, rid, key) return } + if key != "" && r.Method == http.MethodGet { + h.doDownload(w, r, s, rid, key) + return + } http.Error(w, "501 Forbidden", http.StatusNotImplemented) }) @@ -210,3 +217,45 @@ func (h *VersionsHandler) doRestore(w http.ResponseWriter, r *http.Request, s *s } w.WriteHeader(http.StatusNoContent) } + +func (h *VersionsHandler) doDownload(w http.ResponseWriter, r *http.Request, s *svc, rid *provider.ResourceId, key string) { + ctx, span := rtrace.Provider.Tracer("ocdav").Start(r.Context(), "restore") + defer span.End() + + sublog := appctx.GetLogger(ctx).With().Interface("resourceid", rid).Str("key", key).Logger() + + client, err := s.getClient() + if err != nil { + sublog.Error().Err(err).Msg("error getting grpc client") + w.WriteHeader(http.StatusInternalServerError) + return + } + + resStat, err := client.Stat(ctx, &provider.StatRequest{ + Ref: &provider.Reference{ + ResourceId: rid, + }, + }) + + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + + if resStat.Status.Code != rpc.Code_CODE_OK { + HandleErrorStatus(&sublog, w, resStat.Status) + return + } + + fname := filepath.Base(resStat.Info.Path) + + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", fname)) + w.Header().Set("Content-Transfer-Encoding", "binary") + + down := downloader.NewDownloader(client) + down.Download(ctx, resStat.Info.Path, key, w) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } +} diff --git a/pkg/storage/utils/downloader/downloader.go b/pkg/storage/utils/downloader/downloader.go index c424e4fb26..35a9d2520a 100644 --- a/pkg/storage/utils/downloader/downloader.go +++ b/pkg/storage/utils/downloader/downloader.go @@ -27,6 +27,7 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/internal/http/services/datagateway" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rhttp" @@ -35,7 +36,7 @@ import ( // Downloader is the interface implemented by the objects that are able to // download a path into a destination Writer. type Downloader interface { - Download(context.Context, string, io.Writer) error + Download(ctx context.Context, path string, versionKey string, w io.Writer) error } type revaDownloader struct { @@ -61,12 +62,23 @@ func getDownloadProtocol(protocols []*gateway.FileDownloadProtocol, prot string) } // Download downloads a resource given the path to the dst Writer. -func (r *revaDownloader) Download(ctx context.Context, path string, dst io.Writer) error { - downResp, err := r.gtw.InitiateFileDownload(ctx, &provider.InitiateFileDownloadRequest{ +func (r *revaDownloader) Download(ctx context.Context, path, versionKey string, dst io.Writer) error { + req := &provider.InitiateFileDownloadRequest{ Ref: &provider.Reference{ Path: path, }, - }) + } + if versionKey != "" { + req.Opaque = &types.Opaque{ + Map: map[string]*types.OpaqueEntry{ + "version_key": { + Decoder: "plain", + Value: []byte(versionKey), + }, + }, + } + } + downResp, err := r.gtw.InitiateFileDownload(ctx, req) switch { case err != nil: diff --git a/pkg/storage/utils/downloader/mock/downloader_mock.go b/pkg/storage/utils/downloader/mock/downloader_mock.go index 451f69b7a9..8368c9a99a 100644 --- a/pkg/storage/utils/downloader/mock/downloader_mock.go +++ b/pkg/storage/utils/downloader/mock/downloader_mock.go @@ -36,7 +36,7 @@ func NewDownloader() downloader.Downloader { } // Download copies the content of a local file into the dst Writer. -func (m *mockDownloader) Download(ctx context.Context, path string, dst io.Writer) error { +func (m *mockDownloader) Download(ctx context.Context, path, _ string, dst io.Writer) error { f, err := os.Open(path) if err != nil { return err From 2e0adedbf8d58851b7c4ded2db32bf13971ca01d Mon Sep 17 00:00:00 2001 From: gmgigi96 Date: Mon, 3 Apr 2023 15:19:50 +0200 Subject: [PATCH 3/4] fix archiver --- internal/http/services/archiver/manager/archiver.go | 4 ++-- internal/http/services/owncloud/ocdav/versions.go | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/http/services/archiver/manager/archiver.go b/internal/http/services/archiver/manager/archiver.go index 9b73a953dd..1d9ab3fcc0 100644 --- a/internal/http/services/archiver/manager/archiver.go +++ b/internal/http/services/archiver/manager/archiver.go @@ -168,7 +168,7 @@ func (a *Archiver) CreateTar(ctx context.Context, dst io.Writer) error { } if !isDir { - err = a.downloader.Download(ctx, path, w) + err = a.downloader.Download(ctx, path, "", w) if err != nil { return err } @@ -239,7 +239,7 @@ func (a *Archiver) CreateZip(ctx context.Context, dst io.Writer) error { } if !isDir { - err = a.downloader.Download(ctx, path, dst) + err = a.downloader.Download(ctx, path, "", dst) if err != nil { return err } diff --git a/internal/http/services/owncloud/ocdav/versions.go b/internal/http/services/owncloud/ocdav/versions.go index 54ce347c0e..5fe0517da5 100644 --- a/internal/http/services/owncloud/ocdav/versions.go +++ b/internal/http/services/owncloud/ocdav/versions.go @@ -253,8 +253,7 @@ func (h *VersionsHandler) doDownload(w http.ResponseWriter, r *http.Request, s * w.Header().Set("Content-Transfer-Encoding", "binary") down := downloader.NewDownloader(client) - down.Download(ctx, resStat.Info.Path, key, w) - if err != nil { + if err := down.Download(ctx, resStat.Info.Path, key, w); err != nil { w.WriteHeader(http.StatusInternalServerError) return } From c89282ed5eb6c7cd34635774f1c6631a70aa1bbb Mon Sep 17 00:00:00 2001 From: gmgigi96 Date: Mon, 3 Apr 2023 16:36:52 +0200 Subject: [PATCH 4/4] add changelog --- changelog/unreleased/download-file-revisions.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changelog/unreleased/download-file-revisions.md diff --git a/changelog/unreleased/download-file-revisions.md b/changelog/unreleased/download-file-revisions.md new file mode 100644 index 0000000000..ecee71aca4 --- /dev/null +++ b/changelog/unreleased/download-file-revisions.md @@ -0,0 +1,8 @@ +Enhancement: Download file revisions + +Currently it is only possible to restore a file version, +replacing the actual file with the selected version. +This allows an user to download a version file, +without touching/replacing the last version of the file + +https://github.com/cs3org/reva/pull/3766 \ No newline at end of file