From 2469457aaf3f69803df0d44a31c1350a22721efd Mon Sep 17 00:00:00 2001 From: David Christofas Date: Wed, 12 Jan 2022 17:15:05 +0100 Subject: [PATCH] store nodes of a space inside of the space This is still work in progress. In this state the node_test.go run successfully. --- pkg/storage/utils/decomposedfs/grants.go | 8 +- pkg/storage/utils/decomposedfs/lookup.go | 19 +++-- pkg/storage/utils/decomposedfs/node/node.go | 79 ++++++++++++------- .../utils/decomposedfs/node/node_test.go | 6 +- pkg/storage/utils/decomposedfs/recycle.go | 12 +-- .../utils/decomposedfs/recycle_test.go | 2 +- pkg/storage/utils/decomposedfs/revisions.go | 14 ++-- pkg/storage/utils/decomposedfs/spaces.go | 8 +- .../utils/decomposedfs/testhelpers/helpers.go | 22 +++--- pkg/storage/utils/decomposedfs/tree/tree.go | 37 ++++----- .../utils/decomposedfs/tree/tree_test.go | 10 +-- pkg/storage/utils/decomposedfs/upload.go | 8 +- 12 files changed, 132 insertions(+), 93 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/grants.go b/pkg/storage/utils/decomposedfs/grants.go index fd94911e8c..918c164373 100644 --- a/pkg/storage/utils/decomposedfs/grants.go +++ b/pkg/storage/utils/decomposedfs/grants.go @@ -64,10 +64,9 @@ func (fs *Decomposedfs) AddGrant(ctx context.Context, ref *provider.Reference, g return errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name)) } - np := fs.lu.InternalPath(node.ID) e := ace.FromGrant(g) principal, value := e.Marshal() - if err := xattr.Set(np, xattrs.GrantPrefix+principal, value); err != nil { + if err := xattr.Set(node.InternalPath(), xattrs.GrantPrefix+principal, value); err != nil { return err } @@ -104,7 +103,7 @@ func (fs *Decomposedfs) ListGrants(ctx context.Context, ref *provider.Reference) } log := appctx.GetLogger(ctx) - np := fs.lu.InternalPath(node.ID) + np := node.InternalPath() var attrs []string if attrs, err = xattr.List(np); err != nil { log.Error().Err(err).Msg("error listing attributes") @@ -151,8 +150,7 @@ func (fs *Decomposedfs) RemoveGrant(ctx context.Context, ref *provider.Reference attr = xattrs.GrantUserAcePrefix + g.Grantee.GetUserId().OpaqueId } - np := fs.lu.InternalPath(node.ID) - if err = xattr.Remove(np, attr); err != nil { + if err = xattr.Remove(node.InternalPath(), attr); err != nil { return } diff --git a/pkg/storage/utils/decomposedfs/lookup.go b/pkg/storage/utils/decomposedfs/lookup.go index 64c3ee76c0..05f0973d82 100644 --- a/pkg/storage/utils/decomposedfs/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup.go @@ -59,6 +59,7 @@ func (lu *Lookup) NodeFromResource(ctx context.Context, ref *provider.Reference) if err != nil { return nil, err } + n.SpaceID = ref.ResourceId.StorageId } } return n, nil @@ -77,10 +78,18 @@ func (lu *Lookup) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *n // The Resource references the root of a space return lu.NodeFromSpaceID(ctx, id) } - n, err = node.ReadNode(ctx, lu, id.OpaqueId) + var spaceID string + // If the ids are the same then both are the spaceID + if id.StorageId == id.OpaqueId { + spaceID = node.NoSpaceID + } else { + spaceID = id.StorageId + } + n, err = node.ReadNode(ctx, lu, spaceID, id.OpaqueId) if err != nil { return nil, err } + n.SpaceID = spaceID return n, n.FindStorageSpaceRoot() } @@ -102,7 +111,7 @@ func (lu *Lookup) NodeFromSpaceID(ctx context.Context, id *provider.ResourceId) appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[0]).Msg("could not read link, skipping") } - node, err := node.ReadNode(ctx, lu, filepath.Base(target)) + node, err := node.ReadNode(ctx, lu, id.StorageId, filepath.Base(target)) if err != nil { return nil, err } @@ -131,7 +140,7 @@ func (lu *Lookup) Path(ctx context.Context, n *node.Node) (p string, err error) // RootNode returns the root node of the storage func (lu *Lookup) RootNode(ctx context.Context) (*node.Node, error) { - n := node.New("root", "", "", 0, "", nil, lu) + n := node.New(node.NoSpaceID, "root", "", "", 0, "", nil, lu) n.Exists = true return n, nil } @@ -183,8 +192,8 @@ func (lu *Lookup) InternalRoot() string { } // InternalPath returns the internal path for a given ID -func (lu *Lookup) InternalPath(id string) string { - return filepath.Join(lu.Options.Root, "nodes", id) +func (lu *Lookup) InternalPath(spaceID, nodeID string) string { + return filepath.Join(lu.Options.Root, "nodes", spaceID, nodeID) } func (lu *Lookup) mustGetUserLayout(ctx context.Context) string { diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index fd61507a30..518c0bd6c3 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -61,10 +61,13 @@ const ( QuotaUncalculated = "-1" QuotaUnknown = "-2" QuotaUnlimited = "-3" + + NoSpaceID = "" ) // Node represents a node in the tree and provides methods to get a Parent or Child instance type Node struct { + SpaceID string ParentID string ID string Name string @@ -82,17 +85,18 @@ type PathLookup interface { RootNode(ctx context.Context) (node *Node, err error) InternalRoot() string - InternalPath(ID string) string + InternalPath(spaceID, nodeID string) string Path(ctx context.Context, n *Node) (path string, err error) ShareFolder() string } // New returns a new instance of Node -func New(id, parentID, name string, blobsize int64, blobID string, owner *userpb.UserId, lu PathLookup) *Node { +func New(spaceID, id, parentID, name string, blobsize int64, blobID string, owner *userpb.UserId, lu PathLookup) *Node { if blobID == "" { blobID = uuid.New().String() } return &Node{ + SpaceID: spaceID, ID: id, ParentID: parentID, Name: name, @@ -170,10 +174,11 @@ func (n *Node) WriteMetadata(owner *userpb.UserId) (err error) { } // ReadNode creates a new instance from an id and checks if it exists -func ReadNode(ctx context.Context, lu PathLookup, id string) (n *Node, err error) { +func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string) (n *Node, err error) { n = &Node{ - lu: lu, - ID: id, + SpaceID: spaceID, + lu: lu, + ID: nodeID, } nodePath := n.InternalPath() @@ -216,8 +221,7 @@ func ReadNode(ctx context.Context, lu PathLookup, id string) (n *Node, err error return } - // Check if parent exists. Otherwise this node is part of a deleted subtree - _, err = os.Stat(lu.InternalPath(n.ParentID)) + _, err = os.Stat(n.ParentInternalPath()) if err != nil { if isNotFound(err) { return nil, errtypes.NotFound(err.Error()) @@ -240,10 +244,18 @@ func isNotDir(err error) bool { // Child returns the child node with the given name func (n *Node) Child(ctx context.Context, name string) (*Node, error) { + spaceID := n.SpaceID + if spaceID == "" && n.ParentID == "root" { + spaceID = n.ID + } else if n.SpaceRoot != nil { + spaceID = n.SpaceRoot.ID + } link, err := os.Readlink(filepath.Join(n.InternalPath(), filepath.Join("/", name))) if err != nil { if os.IsNotExist(err) || isNotDir(err) { + c := &Node{ + SpaceID: spaceID, lu: n.lu, ParentID: n.ID, Name: name, @@ -255,16 +267,12 @@ func (n *Node) Child(ctx context.Context, name string) (*Node, error) { return nil, errors.Wrap(err, "Decomposedfs: Wrap: readlink error") } - var c *Node - if strings.HasPrefix(link, "../") { - c, err = ReadNode(ctx, n.lu, filepath.Base(link)) - if err != nil { - return nil, errors.Wrap(err, "could not read child node") - } - c.SpaceRoot = n.SpaceRoot - } else { - return nil, fmt.Errorf("Decomposedfs: expected '../ prefix, got' %+v", link) + c, err := ReadNode(ctx, n.lu, spaceID, filepath.Base(link)) + if err != nil { + return nil, errors.Wrap(err, "could not read child node") } + c.SpaceRoot = n.SpaceRoot + c.SpaceID = spaceID return c, nil } @@ -274,13 +282,21 @@ func (n *Node) Parent() (p *Node, err error) { if n.ParentID == "" { return nil, fmt.Errorf("Decomposedfs: root has no parent") } + var spaceID string + if n.SpaceRoot != nil && n.SpaceRoot.ID != n.ParentID { + spaceID = n.SpaceID + } else { + spaceID = "" + } p = &Node{ + SpaceID: spaceID, lu: n.lu, ID: n.ParentID, SpaceRoot: n.SpaceRoot, } - parentPath := n.lu.InternalPath(n.ParentID) + // parentPath := n.lu.InternalPath(spaceID, n.ParentID) + parentPath := p.InternalPath() // lookup parent id in extended attributes var attrBytes []byte @@ -378,7 +394,16 @@ func (n *Node) PermissionSet(ctx context.Context) provider.ResourcePermissions { // InternalPath returns the internal path of the Node func (n *Node) InternalPath() string { - return n.lu.InternalPath(n.ID) + return n.lu.InternalPath(n.SpaceID, n.ID) +} + +func (n *Node) ParentInternalPath() string { + spaceID := n.SpaceID + // If the parent is the space root then we don't need the extra spaceID part in the path. + if n.SpaceID == n.ParentID { + spaceID = "" + } + return n.lu.InternalPath(spaceID, n.ParentID) } // CalculateEtag returns a hash of fileid + tmtime (or mtime) @@ -406,7 +431,7 @@ func calculateEtag(nodeID string, tmTime time.Time) (string, error) { func (n *Node) SetMtime(ctx context.Context, mtime string) error { sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() if mt, err := parseMTime(mtime); err == nil { - nodePath := n.lu.InternalPath(n.ID) + nodePath := n.InternalPath() // updating mtime also updates atime if err := os.Chtimes(nodePath, mt, mt); err != nil { sublog.Error().Err(err). @@ -426,7 +451,7 @@ func (n *Node) SetMtime(ctx context.Context, mtime string) error { // SetEtag sets the temporary etag of a node if it differs from the current etag func (n *Node) SetEtag(ctx context.Context, val string) (err error) { sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() - nodePath := n.lu.InternalPath(n.ID) + nodePath := n.InternalPath() var tmTime time.Time if tmTime, err = n.GetTMTime(); err != nil { // no tmtime, use mtime @@ -471,7 +496,7 @@ func (n *Node) SetEtag(ctx context.Context, val string) (err error) { // obviously this only is secure when the u/s/g/a namespaces are not accessible by users in the filesystem // public tags can be mapped to extended attributes func (n *Node) SetFavorite(uid *userpb.UserId, val string) error { - nodePath := n.lu.InternalPath(n.ID) + nodePath := n.InternalPath() // the favorite flag is specific to the user, so we need to incorporate the userid fa := fmt.Sprintf("%s:%s:%s@%s", xattrs.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) return xattr.Set(nodePath, fa, []byte(val)) @@ -482,7 +507,7 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi sublog := appctx.GetLogger(ctx).With().Interface("node", n).Logger() var fn string - nodePath := n.lu.InternalPath(n.ID) + nodePath := n.InternalPath() var fi os.FileInfo @@ -740,7 +765,7 @@ func readQuotaIntoOpaque(ctx context.Context, nodePath string, ri *provider.Reso // HasPropagation checks if the propagation attribute exists and is set to "1" func (n *Node) HasPropagation() (propagation bool) { - if b, err := xattr.Get(n.lu.InternalPath(n.ID), xattrs.PropagationAttr); err == nil { + if b, err := xattr.Get(n.InternalPath(), xattrs.PropagationAttr); err == nil { return string(b) == "1" } return false @@ -749,7 +774,7 @@ func (n *Node) HasPropagation() (propagation bool) { // GetTMTime reads the tmtime from the extended attributes func (n *Node) GetTMTime() (tmTime time.Time, err error) { var b []byte - if b, err = xattr.Get(n.lu.InternalPath(n.ID), xattrs.TreeMTimeAttr); err != nil { + if b, err = xattr.Get(n.InternalPath(), xattrs.TreeMTimeAttr); err != nil { return } return time.Parse(time.RFC3339Nano, string(b)) @@ -757,7 +782,7 @@ func (n *Node) GetTMTime() (tmTime time.Time, err error) { // SetTMTime writes the tmtime to the extended attributes func (n *Node) SetTMTime(t time.Time) (err error) { - return xattr.Set(n.lu.InternalPath(n.ID), xattrs.TreeMTimeAttr, []byte(t.UTC().Format(time.RFC3339Nano))) + return xattr.Set(n.InternalPath(), xattrs.TreeMTimeAttr, []byte(t.UTC().Format(time.RFC3339Nano))) } // GetTreeSize reads the treesize from the extended attributes @@ -776,12 +801,12 @@ func (n *Node) SetTreeSize(ts uint64) (err error) { // SetChecksum writes the checksum with the given checksum type to the extended attributes func (n *Node) SetChecksum(csType string, h hash.Hash) (err error) { - return xattr.Set(n.lu.InternalPath(n.ID), xattrs.ChecksumPrefix+csType, h.Sum(nil)) + return xattr.Set(n.InternalPath(), xattrs.ChecksumPrefix+csType, h.Sum(nil)) } // UnsetTempEtag removes the temporary etag attribute func (n *Node) UnsetTempEtag() (err error) { - if err = xattr.Remove(n.lu.InternalPath(n.ID), xattrs.TmpEtagAttr); err != nil { + if err = xattr.Remove(n.InternalPath(), xattrs.TmpEtagAttr); err != nil { if e, ok := err.(*xattr.Error); ok && (e.Err.Error() == "no data available" || // darwin e.Err.Error() == "attribute not found") { diff --git a/pkg/storage/utils/decomposedfs/node/node_test.go b/pkg/storage/utils/decomposedfs/node/node_test.go index 210269cf28..4517ad352e 100644 --- a/pkg/storage/utils/decomposedfs/node/node_test.go +++ b/pkg/storage/utils/decomposedfs/node/node_test.go @@ -55,8 +55,8 @@ var _ = Describe("Node", func() { Describe("New", func() { It("generates unique blob ids if none are given", func() { - n1 := node.New(id, "", name, 10, "", env.Owner.Id, env.Lookup) - n2 := node.New(id, "", name, 10, "", env.Owner.Id, env.Lookup) + n1 := node.New(env.SpaceRootRes.StorageId, id, "", name, 10, "", env.Owner.Id, env.Lookup) + n2 := node.New(env.SpaceRootRes.StorageId, id, "", name, 10, "", env.Owner.Id, env.Lookup) Expect(len(n1.BlobID)).To(Equal(36)) Expect(n1.BlobID).ToNot(Equal(n2.BlobID)) @@ -71,7 +71,7 @@ var _ = Describe("Node", func() { }) Expect(err).ToNot(HaveOccurred()) - n, err := node.ReadNode(env.Ctx, env.Lookup, lookupNode.ID) + n, err := node.ReadNode(env.Ctx, env.Lookup, lookupNode.SpaceID, lookupNode.ID) Expect(err).ToNot(HaveOccurred()) Expect(n.BlobID).To(Equal("file1-blobid")) }) diff --git a/pkg/storage/utils/decomposedfs/recycle.go b/pkg/storage/utils/decomposedfs/recycle.go index 1876856f9a..880486aee8 100644 --- a/pkg/storage/utils/decomposedfs/recycle.go +++ b/pkg/storage/utils/decomposedfs/recycle.go @@ -95,7 +95,7 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference return nil, err } else if !md.IsDir() { // this is the case when we want to directly list a file in the trashbin - item, err := fs.createTrashItem(ctx, parentNode, filepath.Dir(relativePath), filepath.Join(trashRoot, key, relativePath)) + item, err := fs.createTrashItem(ctx, spaceID, parentNode, filepath.Dir(relativePath), filepath.Join(trashRoot, key, relativePath)) if err != nil { return items, err } @@ -108,14 +108,14 @@ func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference return nil, err } for i := range names { - if item, err := fs.createTrashItem(ctx, parentNode, relativePath, filepath.Join(trashRoot, key, relativePath, names[i])); err == nil { + if item, err := fs.createTrashItem(ctx, spaceID, parentNode, relativePath, filepath.Join(trashRoot, key, relativePath, names[i])); err == nil { items = append(items, item) } } return items, nil } -func (fs *Decomposedfs) createTrashItem(ctx context.Context, parentNode, intermediatePath, itemPath string) (*provider.RecycleItem, error) { +func (fs *Decomposedfs) createTrashItem(ctx context.Context, spaceID, parentNode, intermediatePath, itemPath string) (*provider.RecycleItem, error) { log := appctx.GetLogger(ctx) trashnode, err := os.Readlink(itemPath) if err != nil { @@ -128,7 +128,7 @@ func (fs *Decomposedfs) createTrashItem(ctx context.Context, parentNode, interme return nil, errors.New("malformed trash link") } - nodePath := fs.lu.InternalPath(filepath.Base(trashnode)) + nodePath := fs.lu.InternalPath(spaceID, filepath.Base(trashnode)) md, err := os.Stat(nodePath) if err != nil { log.Error().Err(err).Str("trashnode", trashnode).Msg("could not stat trash item, skipping") @@ -150,7 +150,7 @@ func (fs *Decomposedfs) createTrashItem(ctx context.Context, parentNode, interme } // lookup origin path in extended attributes - parentPath := fs.lu.InternalPath(filepath.Base(parentNode)) + parentPath := fs.lu.InternalPath(spaceID, filepath.Base(parentNode)) if attrBytes, err := xattr.Get(parentPath, xattrs.TrashOriginAttr); err == nil { item.Ref = &provider.Reference{Path: filepath.Join(string(attrBytes), intermediatePath, filepath.Base(itemPath))} } else { @@ -197,7 +197,7 @@ func (fs *Decomposedfs) listTrashRoot(ctx context.Context, spaceID string) ([]*p continue } - nodePath := fs.lu.InternalPath(filepath.Base(trashnode)) + nodePath := fs.lu.InternalPath(spaceID, filepath.Base(trashnode)) md, err := os.Stat(nodePath) if err != nil { log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("trashnode", trashnode). /*.Interface("parts", parts)*/ Msg("could not stat trash item, skipping") diff --git a/pkg/storage/utils/decomposedfs/recycle_test.go b/pkg/storage/utils/decomposedfs/recycle_test.go index a273024b42..c50967e181 100644 --- a/pkg/storage/utils/decomposedfs/recycle_test.go +++ b/pkg/storage/utils/decomposedfs/recycle_test.go @@ -289,7 +289,7 @@ var _ = Describe("Recycle", func() { Expect(len(items)).To(Equal(1)) // use up 2000 byte quota - _, err = env.CreateTestFile("largefile", "largefile-blobid", 2000, projectID.OpaqueId) + _, err = env.CreateTestFile("largefile", "largefile-blobid", projectID.OpaqueId, projectID.StorageId, 2000) Expect(err).ToNot(HaveOccurred()) err = env.Fs.RestoreRecycleItem(env.Ctx, &provider.Reference{ResourceId: projectID}, items[0].Key, "/", nil) diff --git a/pkg/storage/utils/decomposedfs/revisions.go b/pkg/storage/utils/decomposedfs/revisions.go index 09a0d59c5f..3c81f52b27 100644 --- a/pkg/storage/utils/decomposedfs/revisions.go +++ b/pkg/storage/utils/decomposedfs/revisions.go @@ -101,8 +101,9 @@ func (fs *Decomposedfs) DownloadRevision(ctx context.Context, ref *provider.Refe } log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision") + spaceID := ref.ResourceId.OpaqueId // check if the node is available and has not been deleted - n, err := node.ReadNode(ctx, fs.lu, kp[0]) + n, err := node.ReadNode(ctx, fs.lu, spaceID, kp[0]) if err != nil { return nil, err } @@ -122,7 +123,7 @@ func (fs *Decomposedfs) DownloadRevision(ctx context.Context, ref *provider.Refe return nil, errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name)) } - contentPath := fs.lu.InternalPath(revisionKey) + contentPath := fs.lu.InternalPath(spaceID, revisionKey) r, err := os.Open(contentPath) if err != nil { @@ -145,8 +146,9 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer return errtypes.NotFound(revisionKey) } + spaceID := ref.ResourceId.OpaqueId // check if the node is available and has not been deleted - n, err := node.ReadNode(ctx, fs.lu, kp[0]) + n, err := node.ReadNode(ctx, fs.lu, spaceID, kp[0]) if err != nil { return err } @@ -166,11 +168,11 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer } // move current version to new revision - nodePath := fs.lu.InternalPath(kp[0]) + nodePath := fs.lu.InternalPath(spaceID, kp[0]) var fi os.FileInfo if fi, err = os.Stat(nodePath); err == nil { // versions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries - versionsPath := fs.lu.InternalPath(kp[0] + ".REV." + fi.ModTime().UTC().Format(time.RFC3339Nano)) + versionsPath := fs.lu.InternalPath(spaceID, kp[0]+".REV."+fi.ModTime().UTC().Format(time.RFC3339Nano)) err = os.Rename(nodePath, versionsPath) if err != nil { @@ -179,7 +181,7 @@ func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Refer // copy old revision to current location - revisionPath := fs.lu.InternalPath(revisionKey) + revisionPath := fs.lu.InternalPath(spaceID, revisionKey) if err = os.Rename(revisionPath, nodePath); err != nil { return diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 4b91937793..2ea2ce8009 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -237,7 +237,7 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide continue } - n, err := node.ReadNode(ctx, fs.lu, filepath.Base(target)) + n, err := node.ReadNode(ctx, fs.lu, spaceID, filepath.Base(target)) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Str("id", filepath.Base(target)).Msg("could not read node, skipping") continue @@ -267,7 +267,7 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide if len(matches) <= numShares && nodeID != spaceID { // try node id target := filepath.Join(fs.o.Root, "nodes", nodeID) - n, err := node.ReadNode(ctx, fs.lu, filepath.Base(target)) + n, err := node.ReadNode(ctx, fs.lu, spaceID, filepath.Base(target)) if err != nil { return nil, err } @@ -309,7 +309,7 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[0]).Msg("could not read link, skipping") } - node, err := node.ReadNode(ctx, fs.lu, filepath.Base(target)) + node, err := node.ReadNode(ctx, fs.lu, spaceID, filepath.Base(target)) if err != nil { return nil, err } @@ -356,7 +356,7 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[0]).Msg("could not read link, skipping") } - node, err := node.ReadNode(ctx, fs.lu, filepath.Base(target)) + node, err := node.ReadNode(ctx, fs.lu, spaceID, filepath.Base(target)) if err != nil { return err } diff --git a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go index dc2194bb97..3d004c95e2 100644 --- a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go +++ b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go @@ -136,9 +136,10 @@ func (t *TestEnv) CreateTestDir(name string, parentRef *providerv1beta1.Referenc } // CreateTestFile creates a new file and its metadata and returns a corresponding Node -func (t *TestEnv) CreateTestFile(name, blobID string, blobSize int64, parentID string) (*node.Node, error) { - // Create file in dir1 - file := node.New( +func (t *TestEnv) CreateTestFile(name, blobID, parentID, spaceID string, blobSize int64) (*node.Node, error) { + // Create n in dir1 + n := node.New( + spaceID, uuid.New().String(), parentID, name, @@ -147,22 +148,22 @@ func (t *TestEnv) CreateTestFile(name, blobID string, blobSize int64, parentID s nil, t.Lookup, ) - _, err := os.OpenFile(file.InternalPath(), os.O_CREATE, 0700) + _, err := os.OpenFile(n.InternalPath(), os.O_CREATE, 0700) if err != nil { return nil, err } - err = file.WriteMetadata(t.Owner.Id) + err = n.WriteMetadata(t.Owner.Id) if err != nil { return nil, err } // Link in parent - childNameLink := filepath.Join(t.Lookup.InternalPath(file.ParentID), file.Name) - err = os.Symlink("../"+file.ID, childNameLink) + childNameLink := filepath.Join(n.ParentInternalPath(), n.Name) + err = os.Symlink("../"+n.ID, childNameLink) if err != nil { return nil, err } - return file, err + return n, err } // CreateTestStorageSpace will create a storage space with some directories and files @@ -185,7 +186,8 @@ func (t *TestEnv) CreateTestStorageSpace(typ string, quota *providerv1beta1.Quot ref := buildRef(space.StorageSpace.Id.OpaqueId, "") // the space name attribute is the stop condition in the lookup - h, err := node.ReadNode(t.Ctx, t.Lookup, space.StorageSpace.Id.OpaqueId) + // Since we want to lookup the space node itself we do not provide a spaceID. + h, err := node.ReadNode(t.Ctx, t.Lookup, node.NoSpaceID, space.StorageSpace.Id.OpaqueId) if err != nil { return nil, err } @@ -201,7 +203,7 @@ func (t *TestEnv) CreateTestStorageSpace(typ string, quota *providerv1beta1.Quot } // Create file1 in dir1 - _, err = t.CreateTestFile("file1", "file1-blobid", 1234, dir1.ID) + _, err = t.CreateTestFile("file1", "file1-blobid", dir1.ID, dir1.SpaceID, 1234) if err != nil { return nil, err } diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index 890aac74da..d0baf4d70f 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -63,7 +63,7 @@ type PathLookup interface { RootNode(ctx context.Context) (node *node.Node, err error) InternalRoot() string - InternalPath(ID string) string + InternalPath(spaceID, nodeID string) string Path(ctx context.Context, n *node.Node) (path string, err error) ShareFolder() string } @@ -111,7 +111,7 @@ func (t *Tree) Setup(owner *userpb.UserId, propagateToRoot bool) error { // the root node has an empty name // the root node has no parent - n := node.New("root", "", "", 0, "", nil, t.lookup) + n := node.New(node.NoSpaceID, "root", "", "", 0, "", nil, t.lookup) err := t.createNode(n, owner) if err != nil { return err @@ -174,7 +174,7 @@ func (t *Tree) Setup(owner *userpb.UserId, propagateToRoot bool) error { // linkSpace creates a new symbolic link for a space with the given type st, and node id func (t *Tree) linkSpace(spaceType, spaceID, nodeID string) { spacesPath := filepath.Join(t.root, "spaces", spaceType, spaceID) - expectedTarget := "../../nodes/" + nodeID + expectedTarget := "../../nodes/" + spaceID + nodeID linkTarget, err := os.Readlink(spacesPath) if errors.Is(err, os.ErrNotExist) { err = os.Symlink(expectedTarget, spacesPath) @@ -263,7 +263,7 @@ func (t *Tree) CreateDir(ctx context.Context, n *node.Node) (err error) { } // make child appear in listings - err = os.Symlink("../"+n.ID, filepath.Join(t.lookup.InternalPath(n.ParentID), n.Name)) + err = os.Symlink(n.ID, filepath.Join(n.ParentInternalPath(), n.Name)) if err != nil { // no better way to check unfortunately if !strings.Contains(err.Error(), "file exists") { @@ -298,7 +298,8 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) // are we just renaming (parent stays the same)? if oldNode.ParentID == newNode.ParentID { - parentPath := t.lookup.InternalPath(oldNode.ParentID) + // parentPath := t.lookup.InternalPath(oldNode.SpaceID, oldNode.ParentID) + parentPath := oldNode.ParentInternalPath() // rename child err = os.Rename( @@ -322,8 +323,8 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) // rename child err = os.Rename( - filepath.Join(t.lookup.InternalPath(oldNode.ParentID), oldNode.Name), - filepath.Join(t.lookup.InternalPath(newNode.ParentID), newNode.Name), + filepath.Join(oldNode.ParentInternalPath(), oldNode.Name), + filepath.Join(newNode.ParentInternalPath(), newNode.Name), ) if err != nil { return errors.Wrap(err, "Decomposedfs: could not move child") @@ -376,7 +377,7 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro continue } - child, err := node.ReadNode(ctx, t.lookup, filepath.Base(link)) + child, err := node.ReadNode(ctx, t.lookup, n.SpaceID, filepath.Base(link)) if err != nil { // TODO log continue @@ -394,7 +395,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { deletingSharedResource := ctx.Value(appctx.DeletingSharedResource) if deletingSharedResource != nil && deletingSharedResource.(bool) { - src := filepath.Join(t.lookup.InternalPath(n.ParentID), n.Name) + src := filepath.Join(n.ParentInternalPath(), n.Name) return os.Remove(src) } // Prepare the trash @@ -420,7 +421,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { // first make node appear in the space trash // parent id and name are stored as extended attributes in the node itself trashLink := filepath.Join(t.root, "trash", n.SpaceRoot.ID, n.ID) - err = os.Symlink("../../nodes/"+n.ID+".T."+deletionTime, trashLink) + err = os.Symlink("../../nodes/"+n.SpaceRoot.ID+n.ID+".T."+deletionTime, trashLink) if err != nil { // To roll back changes // TODO unset trashOriginAttr @@ -440,7 +441,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { } // finally remove the entry from the parent dir - src := filepath.Join(t.lookup.InternalPath(n.ParentID), n.Name) + src := filepath.Join(n.ParentInternalPath(), n.Name) err = os.Remove(src) if err != nil { // To roll back changes @@ -483,7 +484,7 @@ func (t *Tree) RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPa } // add the entry for the parent dir - err = os.Symlink("../"+recycleNode.ID, filepath.Join(t.lookup.InternalPath(targetNode.ParentID), targetNode.Name)) + err = os.Symlink("../"+recycleNode.ID, filepath.Join(targetNode.ParentInternalPath(), targetNode.Name)) if err != nil { return err } @@ -776,12 +777,12 @@ func (t *Tree) createNode(n *node.Node, owner *userpb.UserId) (err error) { } // TODO refactor the returned params into Node properties? would make all the path transformations go away... -func (t *Tree) readRecycleItem(ctx context.Context, spaceid, key, path string) (recycleNode *node.Node, trashItem string, deletedNodePath string, origin string, err error) { +func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) (recycleNode *node.Node, trashItem string, deletedNodePath string, origin string, err error) { if key == "" { return nil, "", "", "", errtypes.InternalError("key is empty") } - trashItem = filepath.Join(t.lookup.InternalRoot(), "trash", spaceid, key, path) + trashItem = filepath.Join(t.lookup.InternalRoot(), "trash", spaceID, key, path) var link string link, err = os.Readlink(trashItem) @@ -792,7 +793,7 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceid, key, path string) ( var attrBytes []byte trashNodeID := filepath.Base(link) - deletedNodePath = t.lookup.InternalPath(trashNodeID) + deletedNodePath = t.lookup.InternalPath(spaceID, trashNodeID) owner := &userpb.UserId{} // lookup ownerId in extended attributes @@ -814,7 +815,7 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceid, key, path string) ( return } - recycleNode = node.New(trashNodeID, "", "", 0, "", owner, t.lookup) + recycleNode = node.New(spaceID, trashNodeID, "", "", 0, "", owner, t.lookup) // lookup blobID in extended attributes if attrBytes, err = xattr.Get(deletedNodePath, xattrs.BlobIDAttr); err == nil { recycleNode.BlobID = string(attrBytes) @@ -854,14 +855,14 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceid, key, path string) ( deletedNodeRootPath := deletedNodePath if path != "" && path != "/" { - trashItemRoot := filepath.Join(t.lookup.InternalRoot(), "trash", spaceid, key) + trashItemRoot := filepath.Join(t.lookup.InternalRoot(), "trash", spaceID, key) var rootLink string rootLink, err = os.Readlink(trashItemRoot) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Str("trashItem", trashItem).Msg("error reading trash link") return } - deletedNodeRootPath = t.lookup.InternalPath(filepath.Base(rootLink)) + deletedNodeRootPath = t.lookup.InternalPath(spaceID, filepath.Base(rootLink)) } // lookup origin path in extended attributes if attrBytes, err = xattr.Get(deletedNodeRootPath, xattrs.TrashOriginAttr); err == nil { diff --git a/pkg/storage/utils/decomposedfs/tree/tree_test.go b/pkg/storage/utils/decomposedfs/tree/tree_test.go index ad19150788..001950f005 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree_test.go +++ b/pkg/storage/utils/decomposedfs/tree/tree_test.go @@ -252,7 +252,7 @@ var _ = Describe("Tree", func() { Describe("with TreeTimeAccounting enabled", func() { It("sets the tmtime of the parent", func() { - file, err := env.CreateTestFile("file1", "", 1, dir.ID) + file, err := env.CreateTestFile("file1", "", dir.ID, dir.SpaceID, 1) Expect(err).ToNot(HaveOccurred()) perms := node.OwnerPermissions() @@ -270,7 +270,7 @@ var _ = Describe("Tree", func() { Describe("with TreeSizeAccounting enabled", func() { It("calculates the size", func() { - file, err := env.CreateTestFile("file1", "", 1, dir.ID) + file, err := env.CreateTestFile("file1", "", dir.ID, dir.SpaceID, 1) Expect(err).ToNot(HaveOccurred()) err = env.Tree.Propagate(env.Ctx, file) @@ -281,9 +281,9 @@ var _ = Describe("Tree", func() { }) It("considers all files", func() { - _, err := env.CreateTestFile("file1", "", 1, dir.ID) + _, err := env.CreateTestFile("file1", "", dir.ID, dir.SpaceID, 1) Expect(err).ToNot(HaveOccurred()) - file2, err := env.CreateTestFile("file2", "", 100, dir.ID) + file2, err := env.CreateTestFile("file2", "", dir.ID, dir.SpaceID, 100) Expect(err).ToNot(HaveOccurred()) err = env.Tree.Propagate(env.Ctx, file2) @@ -299,7 +299,7 @@ var _ = Describe("Tree", func() { err = subdir.SetTreeSize(uint64(200)) Expect(err).ToNot(HaveOccurred()) - file, err := env.CreateTestFile("file1", "", 1, dir.ID) + file, err := env.CreateTestFile("file1", "", dir.ID, dir.SpaceID, 1) Expect(err).ToNot(HaveOccurred()) err = env.Tree.Propagate(env.Ctx, file) diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 8218fae25a..1b631e42ae 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -458,7 +458,9 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { return } + spaceID := upload.info.Storage["SpaceRoot"] n := node.New( + spaceID, upload.info.Storage["NodeId"], upload.info.Storage["NodeParentId"], upload.info.Storage["NodeName"], @@ -467,7 +469,7 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { nil, upload.fs.lu, ) - n.SpaceRoot = node.New(upload.info.Storage["SpaceRoot"], "", "", 0, "", nil, upload.fs.lu) + n.SpaceRoot = node.New(node.NoSpaceID, spaceID, "", "", 0, "", nil, upload.fs.lu) _, err = node.CheckQuota(n.SpaceRoot, uint64(fi.Size())) if err != nil { @@ -537,7 +539,7 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { if fi, err = os.Stat(targetPath); err == nil { // FIXME move versioning to blobs ... no need to copy all the metadata! well ... it does if we want to version metadata... // versions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries - versionsPath = upload.fs.lu.InternalPath(n.ID + ".REV." + fi.ModTime().UTC().Format(time.RFC3339Nano)) + versionsPath = upload.fs.lu.InternalPath(spaceID, n.ID+".REV."+fi.ModTime().UTC().Format(time.RFC3339Nano)) // This move drops all metadata!!! We copy it below with CopyMetadata // FIXME the node must remain the same. otherwise we might restore share metadata @@ -608,7 +610,7 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { } // link child name to parent if it is new - childNameLink := filepath.Join(upload.fs.lu.InternalPath(n.ParentID), n.Name) + childNameLink := filepath.Join(n.ParentInternalPath(), n.Name) var link string link, err = os.Readlink(childNameLink) if err == nil && link != "../"+n.ID {