From 283cacb874a36e6d9267aae118b5612bc8caf2a7 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 28 Dec 2022 16:11:43 +0100 Subject: [PATCH 1/3] add mtime when requested Signed-off-by: jkoberg --- pkg/storage/utils/decomposedfs/upload/processing.go | 7 +++++++ pkg/storage/utils/decomposedfs/upload/upload.go | 8 -------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/upload/processing.go b/pkg/storage/utils/decomposedfs/upload/processing.go index 73ff6ec4d8..09b574002f 100644 --- a/pkg/storage/utils/decomposedfs/upload/processing.go +++ b/pkg/storage/utils/decomposedfs/upload/processing.go @@ -291,6 +291,13 @@ func CreateNodeForUpload(upload *Upload, initAttrs map[string]string) (*node.Nod return nil, errors.Wrap(err, "Decomposedfs: could not write metadata") } + // overwrite mtime if requested + if upload.Info.MetaData["mtime"] != "" { + if err := n.SetMtimeString(upload.Info.MetaData["mtime"]); err != nil { + return nil, err + } + } + // update nodeid for later upload.Info.Storage["NodeId"] = n.ID if err := upload.writeInfo(); err != nil { diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index a533694092..2abd33ce0c 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -359,14 +359,6 @@ func (upload *Upload) Finalize() (err error) { Str("spaceID", upload.Info.Storage["SpaceRoot"]). Logger() - // tests sometimes set the mtime - if upload.Info.MetaData["mtime"] != "" { - if err := n.SetMtimeString(upload.Info.MetaData["mtime"]); err != nil { - sublog.Err(err).Interface("info", upload.Info).Msg("Decomposedfs: could not set mtime metadata") - return err - } - } - // some clients need the etag in the upload metadata fi, err := os.Stat(n.InternalPath()) if err != nil { From b917249d0e7a76f3502c3f9530ec276a46b1a099 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 28 Dec 2022 16:22:07 +0100 Subject: [PATCH 2/3] add etag to uploadinfo Signed-off-by: jkoberg --- .../utils/decomposedfs/upload/processing.go | 7 ++++ .../utils/decomposedfs/upload/upload.go | 35 ++++--------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/upload/processing.go b/pkg/storage/utils/decomposedfs/upload/processing.go index 09b574002f..7b753ae7f5 100644 --- a/pkg/storage/utils/decomposedfs/upload/processing.go +++ b/pkg/storage/utils/decomposedfs/upload/processing.go @@ -298,6 +298,13 @@ func CreateNodeForUpload(upload *Upload, initAttrs map[string]string) (*node.Nod } } + // add etag to metadata + nfi, err := os.Stat(n.InternalPath()) + if err != nil { + return nil, err + } + upload.Info.MetaData["etag"], _ = node.CalculateEtag(n.ID, nfi.ModTime()) + // update nodeid for later upload.Info.Storage["NodeId"] = n.ID if err := upload.writeInfo(); err != nil { diff --git a/pkg/storage/utils/decomposedfs/upload/upload.go b/pkg/storage/utils/decomposedfs/upload/upload.go index 2abd33ce0c..3d689c064c 100644 --- a/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/pkg/storage/utils/decomposedfs/upload/upload.go @@ -263,15 +263,13 @@ func (upload *Upload) FinishUpload(_ context.Context) error { } } - if upload.async { - // handle postprocessing asynchronously but inform there is something in progress - return upload.tp.Propagate(upload.Ctx, n, upload.sizeDiff) - } - - err = upload.Finalize() - Cleanup(upload, err != nil, false) - if err != nil { - return err + if !upload.async { + // handle postprocessing synchronously + err = upload.Finalize() + Cleanup(upload, err != nil, false) + if err != nil { + return err + } } return upload.tp.Propagate(upload.Ctx, n, upload.sizeDiff) @@ -347,25 +345,6 @@ func (upload *Upload) Finalize() (err error) { return errors.Wrap(err, "failed to upload file to blostore") } - if upload.async { - return nil - } - - sublog := appctx.GetLogger(upload.Ctx). - With(). - Interface("info", upload.Info). - Str("binPath", upload.binPath). - Str("targetPath", n.InternalPath()). - Str("spaceID", upload.Info.Storage["SpaceRoot"]). - Logger() - - // some clients need the etag in the upload metadata - fi, err := os.Stat(n.InternalPath()) - if err != nil { - sublog.Err(err).Interface("info", upload.Info).Str("path", n.InternalPath()).Msg("Decomposedfs: could not stat file") - return err - } - upload.Info.MetaData["etag"], _ = node.CalculateEtag(n.ID, fi.ModTime()) return nil } From 8ea0228200108be3cab6ec552ac70f23a9bb787c Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 28 Dec 2022 16:24:29 +0100 Subject: [PATCH 3/3] changelog Signed-off-by: jkoberg --- changelog/unreleased/async-uploads-improvements.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changelog/unreleased/async-uploads-improvements.md b/changelog/unreleased/async-uploads-improvements.md index 4aea822007..453ab2963c 100644 --- a/changelog/unreleased/async-uploads-improvements.md +++ b/changelog/unreleased/async-uploads-improvements.md @@ -1,6 +1,9 @@ Enhancement: Async Upload Improvements Collection of smaller fixes and quality of life improvements for async postprocessing, especially the upload part. -Contains unit tests, 0 byte uploads, adjusted endpoint responses and more +Contains unit tests, 0 byte uploads, adjusted endpoint responses and more. Adjust mtime when requested from upload. +Add etag to upload info. https://github.com/cs3org/reva/pull/3556 + +https://github.com/cs3org/reva/pull/3571