From 3298bbffc281b25c7e36188b404eb08612418b0d Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Wed, 5 Jun 2024 17:27:32 +0200 Subject: [PATCH] cephfs: initial implementation of recycle bin functionality --- changelog/unreleased/ceph-recycle.md | 7 ++ go.mod | 1 + go.sum | 1 + pkg/storage/fs/cephfs/cephfs.go | 134 ++++++++++++++++++++++++--- pkg/storage/fs/cephfs/options.go | 25 ++++- 5 files changed, 156 insertions(+), 12 deletions(-) create mode 100644 changelog/unreleased/ceph-recycle.md diff --git a/changelog/unreleased/ceph-recycle.md b/changelog/unreleased/ceph-recycle.md new file mode 100644 index 0000000000..49b1549356 --- /dev/null +++ b/changelog/unreleased/ceph-recycle.md @@ -0,0 +1,7 @@ +Enhancement: recycle bin functionality for cephfs + +This implementation is modeled after the CERN-deployed WinSpaces, +where a folder within each space is designated as the recycle folder +and organized by dates. + +https://github.com/cs3org/reva/pull/4713 diff --git a/go.mod b/go.mod index 4a26493062..0c81446523 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/go-playground/validator/v10 v10.19.0 github.com/go-sql-driver/mysql v1.8.0 github.com/gofrs/uuid v4.4.0+incompatible + github.com/gogo/protobuf v1.3.2 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/protobuf v1.5.4 github.com/gomodule/redigo v1.9.2 diff --git a/go.sum b/go.sum index 78ba809f8d..94d515a150 100644 --- a/go.sum +++ b/go.sum @@ -1016,6 +1016,7 @@ github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFG github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= diff --git a/pkg/storage/fs/cephfs/cephfs.go b/pkg/storage/fs/cephfs/cephfs.go index 1f32df6bb2..c1e0bcb13c 100644 --- a/pkg/storage/fs/cephfs/cephfs.go +++ b/pkg/storage/fs/cephfs/cephfs.go @@ -38,6 +38,7 @@ import ( goceph "github.com/ceph/go-ceph/cephfs" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" typepb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/storage" @@ -171,6 +172,21 @@ func (fs *cephfs) CreateDir(ctx context.Context, ref *provider.Reference) error return getRevaError(err) } +func getRecycleTargetFromPath(path string, recyclePath string, recyclePathDepth int) (string, error) { + // Tokenize the given (absolute) path + components := strings.Split(filepath.Clean(string(filepath.Separator)+path), string(filepath.Separator)) + if recyclePathDepth > len(components)-1 { + return "", errors.New("path is too short") + } + + // And construct the target by injecting the recyclePath at the required depth + var target []string = []string{string(filepath.Separator)} + target = append(target, components[:recyclePathDepth+1]...) + target = append(target, recyclePath, time.Now().Format("2006/01/02")) + target = append(target, components[recyclePathDepth+1:]...) + return filepath.Join(target...), nil +} + func (fs *cephfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { var path string user := fs.makeUser(ctx) @@ -180,10 +196,17 @@ func (fs *cephfs) Delete(ctx context.Context, ref *provider.Reference) (err erro } user.op(func(cv *cacheVal) { - if err = cv.mount.Unlink(path); err != nil && err.Error() == errIsADirectory { - err = cv.mount.RemoveDir(path) + if fs.conf.RecyclePath != "" { + // Recycle bin is configured, move to recycle as opposed to unlink + targetPath, err := getRecycleTargetFromPath(path, fs.conf.RecyclePath, fs.conf.RecyclePathDepth) + if err == nil { + err = cv.mount.Rename(path, targetPath) + } + } else { + if err = cv.mount.Unlink(path); err != nil && err.Error() == errIsADirectory { + err = cv.mount.RemoveDir(path) + } } - //TODO(tmourati): Add entry id logic }) @@ -597,24 +620,113 @@ func (fs *cephfs) TouchFile(ctx context.Context, ref *provider.Reference) error return getRevaError(err) } -func (fs *cephfs) EmptyRecycle(ctx context.Context) error { - return errtypes.NotSupported("unimplemented") -} +func (fs *cephfs) listDeletedEntries(ctx context.Context, maxentries int, basePath string, from, to time.Time) (res []*provider.RecycleItem, err error) { + res = []*provider.RecycleItem{} + user := fs.makeUser(ctx) + count := 0 + rootRecyclePath := filepath.Join(basePath, fs.conf.RecyclePath) + for d := to; !d.Before(from); d = d.AddDate(0, 0, -1) { -func (fs *cephfs) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (r *provider.CreateStorageSpaceResponse, err error) { - return nil, errtypes.NotSupported("unimplemented") + user.op(func(cv *cacheVal) { + var dir *goceph.Directory + if dir, err = cv.mount.OpenDir(filepath.Join(rootRecyclePath, d.Format("2006/01/02"))); err != nil { + return + } + defer closeDir(dir) + + var entry *goceph.DirEntryPlus + for entry, err = dir.ReadDirPlus(goceph.StatxBasicStats, 0); entry != nil && err == nil; entry, err = dir.ReadDirPlus(goceph.StatxBasicStats, 0) { + //TODO(lopresti) validate content of entry.Name() here. + targetPath := filepath.Join(basePath, entry.Name()) + stat := entry.Statx() + res = append(res, &provider.RecycleItem{ + Ref: &provider.Reference{Path: targetPath}, + Key: filepath.Join(rootRecyclePath, targetPath), + Size: stat.Size, + DeletionTime: &typesv1beta1.Timestamp{ + Seconds: uint64(stat.Mtime.Sec), + Nanos: uint32(stat.Mtime.Nsec), + }, + }) + + count += 1 + if count > maxentries { + err = errtypes.BadRequest("list too long") + return + } + } + }) + } + return res, err } func (fs *cephfs) ListRecycle(ctx context.Context, basePath, key, relativePath string, from, to *typepb.Timestamp) ([]*provider.RecycleItem, error) { - return nil, errtypes.NotSupported("unimplemented") + md, err := fs.GetMD(ctx, &provider.Reference{Path: basePath}, nil) + if err != nil { + return nil, err + } + if !md.PermissionSet.ListRecycle { + return nil, errtypes.PermissionDenied("cephfs: user doesn't have permissions to restore recycled items") + } + + var dateFrom, dateTo time.Time + if from != nil && to != nil { + dateFrom = time.Unix(int64(from.Seconds), 0) + dateTo = time.Unix(int64(to.Seconds), 0) + if dateFrom.AddDate(0, 0, fs.conf.MaxDaysInRecycleList).Before(dateTo) { + return nil, errtypes.BadRequest("cephfs: too many days requested in listing the recycle bin") + } + } else { + // if no date range was given, list up to two days ago + dateTo = time.Now() + dateFrom = dateTo.AddDate(0, 0, -2) + } + + sublog := appctx.GetLogger(ctx).With().Logger() + sublog.Debug().Time("from", dateFrom).Time("to", dateTo).Msg("executing ListDeletedEntries") + recycleEntries, err := fs.listDeletedEntries(ctx, fs.conf.MaxRecycleEntries, basePath, dateFrom, dateTo) + if err != nil { + switch err.(type) { + case errtypes.IsBadRequest: + return nil, errtypes.BadRequest("cephfs: too many entries found in listing the recycle bin") + default: + return nil, errors.Wrap(err, "cephfs: error listing deleted entries") + } + } + return recycleEntries, nil } func (fs *cephfs) RestoreRecycleItem(ctx context.Context, basePath, key, relativePath string, restoreRef *provider.Reference) error { - return errtypes.NotSupported("unimplemented") + user := fs.makeUser(ctx) + md, err := fs.GetMD(ctx, &provider.Reference{Path: basePath}, nil) + if err != nil { + return err + } + if !md.PermissionSet.RestoreRecycleItem { + return errtypes.PermissionDenied("cephfs: user doesn't have permissions to restore recycled items") + } + + user.op(func(cv *cacheVal) { + //TODO(lopresti) validate content of basePath and relativePath. Key is expected to contain the recycled path + if err = cv.mount.Rename(key, filepath.Join(basePath, relativePath)); err != nil { + return + } + //TODO(tmourati): Add entry id logic, handle already moved file error + }) + + return getRevaError(err) } func (fs *cephfs) PurgeRecycleItem(ctx context.Context, basePath, key, relativePath string) error { - return errtypes.NotSupported("unimplemented") + return errtypes.NotSupported("cephfs: operation not supported") +} + +func (fs *cephfs) EmptyRecycle(ctx context.Context) error { + return errtypes.NotSupported("cephfs: operation not supported") +} + +func (fs *cephfs) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (r *provider.CreateStorageSpaceResponse, err error) { + return nil, errtypes.NotSupported("unimplemented") } func (fs *cephfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter) ([]*provider.StorageSpace, error) { diff --git a/pkg/storage/fs/cephfs/options.go b/pkg/storage/fs/cephfs/options.go index c33e76eb89..1fe587c34f 100644 --- a/pkg/storage/fs/cephfs/options.go +++ b/pkg/storage/fs/cephfs/options.go @@ -44,7 +44,19 @@ type Options struct { DirPerms uint32 `mapstructure:"dir_perms"` FilePerms uint32 `mapstructure:"file_perms"` UserQuotaBytes uint64 `mapstructure:"user_quota_bytes"` - HiddenDirs map[string]bool + // Path of the recycle bin. If empty, recycling is disabled. + RecyclePath string `mapstructure:"recycle_path"` + // Depth of the Recycle bin location, that is after how many path components + // the recycle path is located: this allows supporting recycles such as + // /top-level/s/space/.recycle with a depth = 3. Defaults to 0. + RecyclePathDepth int `mapstructure:"recycle_path_depth"` + // Maximum entries count a ListRecycle call may return: if exceeded, ListRecycle + // will return a BadRequest error + MaxRecycleEntries int `mapstructure:"max_recycle_entries"` + // Maximum time span in days a ListRecycle call may return: if exceeded, ListRecycle + // will override the "to" date with "from" + this value + MaxDaysInRecycleList int `mapstructure:"max_days_in_recycle_list"` + HiddenDirs map[string]bool } func (c *Options) ApplyDefaults() { @@ -102,6 +114,9 @@ func (c *Options) ApplyDefaults() { "..": true, removeLeadingSlash(c.ShadowFolder): true, } + if c.RecyclePath != "" { + c.HiddenDirs[c.RecyclePath] = true + } if c.DirPerms == 0 { c.DirPerms = dirPermDefault @@ -114,4 +129,12 @@ func (c *Options) ApplyDefaults() { if c.UserQuotaBytes == 0 { c.UserQuotaBytes = 50000000000 } + + if c.MaxDaysInRecycleList == 0 { + c.MaxDaysInRecycleList = 14 + } + + if c.MaxRecycleEntries == 0 { + c.MaxRecycleEntries = 2000 + } }