From e4c1191f5d0571a00cd897e8e54fa8801752cc39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 16 Feb 2022 13:55:24 +0000 Subject: [PATCH] introduce new layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../utils/decomposedfs/decomposedfs.go | 54 +----- pkg/storage/utils/decomposedfs/lookup.go | 10 +- pkg/storage/utils/decomposedfs/node/node.go | 39 ++-- pkg/storage/utils/decomposedfs/spaces.go | 180 ++++++++++-------- pkg/storage/utils/decomposedfs/tree/tree.go | 137 +++++++------ pkg/storage/utils/decomposedfs/upload.go | 11 +- 6 files changed, 214 insertions(+), 217 deletions(-) diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 7c7c0dce3ab..a3befde1b49 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -34,6 +34,7 @@ import ( "syscall" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/appctx" ctxpkg "github.com/cs3org/reva/pkg/ctx" @@ -204,57 +205,18 @@ func (fs *Decomposedfs) CreateHome(ctx context.Context) (err error) { return errtypes.NotSupported("Decomposedfs: CreateHome() home supported disabled") } - var n, h *node.Node - if n, err = fs.lu.RootNode(ctx); err != nil { - return - } - h, err = fs.lu.WalkPath(ctx, n, fs.lu.mustGetUserLayout(ctx), false, func(ctx context.Context, n *node.Node) error { - if !n.Exists { - if err := fs.tp.CreateDir(ctx, n); err != nil { - return err - } - } - return nil + u := ctxpkg.ContextMustGetUser(ctx) + res, err := fs.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{ + Type: spaceTypePersonal, + Owner: u, }) - - // make sure to delete the created directory if things go wrong - defer func() { - if err != nil { - // do not catch the error to not shadow the original error - if tmpErr := fs.tp.Delete(ctx, n); tmpErr != nil { - appctx.GetLogger(ctx).Error().Err(tmpErr).Msg("Can not revert file system change after error") - } - } - }() - if err != nil { - return - } - - // update the owner - u := ctxpkg.ContextMustGetUser(ctx) - if err = h.WriteAllNodeMetadata(u.Id); err != nil { - return - } - - if fs.o.TreeTimeAccounting || fs.o.TreeSizeAccounting { - // mark the home node as the end of propagation - if err = h.SetMetadata(xattrs.PropagationAttr, "1"); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", h).Msg("could not mark home as propagation root") - return - } - } - - if err := h.SetMetadata(xattrs.SpaceNameAttr, u.DisplayName); err != nil { return err } - - // add storage space - if err := fs.createStorageSpace(ctx, spaceTypePersonal, h.ID); err != nil { - return err + if res.Status.Code != rpcv1beta1.Code_CODE_OK { + return errtypes.NewErrtypeFromStatus(res.Status) } - - return + return nil } // The os not exists error is buried inside the xattr error, diff --git a/pkg/storage/utils/decomposedfs/lookup.go b/pkg/storage/utils/decomposedfs/lookup.go index 9f6f5cbfe10..bbe70c385c0 100644 --- a/pkg/storage/utils/decomposedfs/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup.go @@ -76,18 +76,10 @@ func (lu *Lookup) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *n // The Resource references the root of a space return lu.NodeFromSpaceID(ctx, id) } - var spaceID string - // If the ids are the same then both are the spaceID - if id.StorageId == id.OpaqueId { - spaceID = node.NoSpaceID - } else { - spaceID = id.StorageId - } - n, err = node.ReadNode(ctx, lu, spaceID, id.OpaqueId) + n, err = node.ReadNode(ctx, lu, id.StorageId, id.OpaqueId) if err != nil { return nil, err } - n.SpaceID = spaceID return n, n.FindStorageSpaceRoot() } diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 4f0963df858..ce05db4ae3f 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -241,6 +241,16 @@ func isNotDir(err error) bool { return false } +func readChildNodeFromLink(path string) (string, error) { + link, err := os.Readlink(path) + if err != nil { + return "", err + } + nodeID := strings.TrimLeft(link, "/.") + nodeID = strings.ReplaceAll(nodeID, "/", "") + return nodeID, nil +} + // Child returns the child node with the given name func (n *Node) Child(ctx context.Context, name string) (*Node, error) { spaceID := n.SpaceID @@ -249,7 +259,7 @@ func (n *Node) Child(ctx context.Context, name string) (*Node, error) { } else if n.SpaceRoot != nil { spaceID = n.SpaceRoot.ID } - link, err := os.Readlink(filepath.Join(n.InternalPath(), filepath.Join("/", name))) + nodeID, err := readChildNodeFromLink(filepath.Join(n.InternalPath(), name)) if err != nil { if os.IsNotExist(err) || isNotDir(err) { @@ -267,17 +277,11 @@ func (n *Node) Child(ctx context.Context, name string) (*Node, error) { } var c *Node - if strings.HasPrefix(link, "../") { - c, err = ReadNode(ctx, n.lu, spaceID, filepath.Base(link)) - if err != nil { - return nil, errors.Wrap(err, "could not read child node") - } - c.SpaceRoot = n.SpaceRoot - } else { - return nil, fmt.Errorf("decomposedfs: expected '../ prefix, got' %+v", link) + c, err = ReadNode(ctx, n.lu, spaceID, nodeID) + if err != nil { + return nil, errors.Wrap(err, "could not read child node") } c.SpaceRoot = n.SpaceRoot - c.SpaceID = spaceID return c, nil } @@ -287,14 +291,8 @@ func (n *Node) Parent() (p *Node, err error) { if n.ParentID == "" { return nil, fmt.Errorf("decomposedfs: root has no parent") } - var spaceID string - if n.SpaceRoot != nil && n.SpaceRoot.ID != n.ParentID { - spaceID = n.SpaceID - } else { - spaceID = "" - } p = &Node{ - SpaceID: spaceID, + SpaceID: n.SpaceID, lu: n.lu, ID: n.ParentID, SpaceRoot: n.SpaceRoot, @@ -401,12 +399,7 @@ func (n *Node) InternalPath() string { } func (n *Node) ParentInternalPath() string { - spaceID := n.SpaceID - // If the parent is the space root then we don't need the extra spaceID part in the path. - if n.SpaceID == n.ParentID { - spaceID = "" - } - return n.lu.InternalPath(spaceID, n.ParentID) + return n.lu.InternalPath(n.SpaceID, n.ParentID) } // LockFilePath returns the internal path of the lock file of the node diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 2b73447e4da..5bef609c2a8 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -44,6 +44,7 @@ import ( "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" "github.com/cs3org/reva/pkg/utils" "github.com/google/uuid" + "github.com/pkg/errors" ) const ( @@ -56,11 +57,6 @@ const ( // CreateStorageSpace creates a storage space func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) { - // spaces will be located by default in the root of the storage. - r, err := fs.lu.RootNode(ctx) - if err != nil { - return nil, err - } // "everything is a resource" this is the unique ID for the Space resource. spaceID := uuid.New().String() @@ -80,30 +76,29 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr // TODO enforce a uuid? // TODO clarify if we want to enforce a single personal storage space or if we want to allow sending the spaceid if req.Type == spaceTypePersonal { - spaceID = req.Owner.Id.OpaqueId + spaceID = req.GetOwner().GetId().GetOpaqueId() } - n, err := r.Child(ctx, spaceID) - if err != nil { - return nil, err - } - - if n.Exists { + root, err := node.ReadNode(ctx, fs.lu, spaceID, spaceID) + if err == nil && root.Exists { return nil, errtypes.AlreadyExists("decomposedfs: spaces: space already exists") } - // spaceid and nodeid must be the same - // TODO enforce a uuid? - n.ID = spaceID + // create a directory node + rootPath := root.InternalPath() + if err = os.MkdirAll(rootPath, 0700); err != nil { + return nil, errors.Wrap(err, "decomposedfs: error creating node") + } - if err := fs.tp.CreateDir(ctx, n); err != nil { + root.WriteAllNodeMetadata(req.GetOwner().GetId()) + if err := root.WriteAllNodeMetadata(req.GetOwner().GetId()); err != nil { return nil, err } // always enable propagation on the storage space root // mark the space root node as the end of propagation - if err = n.SetMetadata(xattrs.PropagationAttr, "1"); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not mark node to propagate") + if err = root.SetMetadata(xattrs.PropagationAttr, "1"); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Interface("node", root).Msg("could not mark space root node to propagate") return nil, err } @@ -117,11 +112,11 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr ownerID = &userv1beta1.UserId{} } - if err := n.ChangeOwner(ownerID); err != nil { + if err := root.ChangeOwner(ownerID); err != nil { return nil, err } - err = fs.createStorageSpace(ctx, req.Type, n.ID) + err = fs.createStorageSpace(ctx, req.Type, root.ID) if err != nil { return nil, err } @@ -136,7 +131,7 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr if description != "" { metadata[xattrs.SpaceDescriptionAttr] = description } - if err := xattrs.SetMultiple(n.InternalPath(), metadata); err != nil { + if err := xattrs.SetMultiple(root.InternalPath(), metadata); err != nil { return nil, err } @@ -159,7 +154,7 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr return nil, err } - space, err := fs.storageSpaceFromNode(ctx, n, "*", n.InternalPath(), false) + space, err := fs.storageSpaceFromNode(ctx, root, "*", root.InternalPath(), false) if err != nil { return nil, err } @@ -173,6 +168,42 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr return resp, nil } +func (fs *Decomposedfs) canListAllSpaces(ctx context.Context) bool { + client, err := pool.GetGatewayServiceClient(fs.o.GatewayAddr) + if err != nil { + return false + } + + user := ctxpkg.ContextMustGetUser(ctx) + checkRes, err := client.CheckPermission(ctx, &permissionsv1beta1.CheckPermissionRequest{ + Permission: "list-all-spaces", + SubjectRef: &permissionsv1beta1.SubjectReference{ + Spec: &permissionsv1beta1.SubjectReference_UserId{ + UserId: user.Id, + }, + }, + }) + if err != nil { + return false + } + + return checkRes.Status.Code == v1beta11.Code_CODE_OK +} + +func readSpaceAndNodeFromSpaceTypeLink(path string) (string, string, error) { + link, err := os.Readlink(path) + if err != nil { + return "", "", err + } + // TODO use filepath.Separator to support windows + link = strings.ReplaceAll(link, "/", "") + // ../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51 + if link[0:10] != "....spaces" || link[46:51] != "nodes" { + return "", "", errtypes.InternalError("malformed link") + } + return link[10:46], link[51:], nil +} + // ListStorageSpaces returns a list of StorageSpaces. // The list can be filtered by space type or space id. // Spaces are persisted with symlinks in /spaces// pointing to ../../nodes/, the root node of the space @@ -217,15 +248,38 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide spaceTypes = []string{"*"} } + canListAllSpaces := fs.canListAllSpaces(ctx) + spaces := []*provider.StorageSpace{} // build the glob path, eg. // /path/to/root/spaces/{spaceType}/{spaceId} // /path/to/root/spaces/personal/nodeid // /path/to/root/spaces/shared/nodeid + if spaceID != spaceIDAny && nodeID != spaceIDAny { + // try directly reading the node + n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID) + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("id", nodeID).Msg("could not read node") + return nil, err + } + space, err := fs.storageSpaceFromNode(ctx, n, spaceTypeAny, n.InternalPath(), canListAllSpaces) + if err != nil { + return nil, err + } + // filter space types + for _, spaceType := range spaceTypes { + if spaceType == "*" || spaceType == space.SpaceType { + spaces = append(spaces, space) + } + } + + return spaces, nil + } + matches := []string{} for _, spaceType := range spaceTypes { - path := filepath.Join(fs.o.Root, "spaces", spaceType, nodeID) + path := filepath.Join(fs.o.Root, "spacetypes", spaceType, nodeID) m, err := filepath.Glob(path) if err != nil { return nil, err @@ -245,41 +299,19 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide // the personal spaces must also use the nodeid and not the name numShares := 0 - client, err := pool.GetGatewayServiceClient(fs.o.GatewayAddr) - if err != nil { - return nil, err - } - - user := ctxpkg.ContextMustGetUser(ctx) - checkRes, err := client.CheckPermission(ctx, &permissionsv1beta1.CheckPermissionRequest{ - Permission: "list-all-spaces", - SubjectRef: &permissionsv1beta1.SubjectReference{ - Spec: &permissionsv1beta1.SubjectReference_UserId{ - UserId: user.Id, - }, - }, - }) - if err != nil { - return nil, err - } - - canListAllSpaces := false - if checkRes.Status.Code == v1beta11.Code_CODE_OK { - canListAllSpaces = true - } for i := range matches { - var target string var err error // always read link in case storage space id != node id - if target, err = os.Readlink(matches[i]); err != nil { + spaceID, nodeID, err = readSpaceAndNodeFromSpaceTypeLink(matches[i]) + if err != nil { appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[i]).Msg("could not read link, skipping") continue } - n, err := node.ReadNode(ctx, fs.lu, spaceID, filepath.Base(target)) + n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID) if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("id", filepath.Base(target)).Msg("could not read node, skipping") + appctx.GetLogger(ctx).Error().Err(err).Str("id", nodeID).Msg("could not read node, skipping") continue } @@ -306,8 +338,7 @@ func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provide // if there are no matches (or they happened to be spaces for the owner) and the node is a child return a space if len(matches) <= numShares && nodeID != spaceID { // try node id - target := filepath.Join(fs.o.Root, "nodes", nodeID) - n, err := node.ReadNode(ctx, fs.lu, spaceID, filepath.Base(target)) + n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID) if err != nil { return nil, err } @@ -335,7 +366,7 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up _, spaceID, _ := utils.SplitStorageSpaceID(space.Id.OpaqueId) if restore { - matches, err := filepath.Glob(filepath.Join(fs.o.Root, "spaces", spaceTypeAny, spaceID)) + matches, err := filepath.Glob(filepath.Join(fs.o.Root, "spacetypes", spaceTypeAny, spaceID)) if err != nil { return nil, err } @@ -354,8 +385,9 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up if err != nil { appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[0]).Msg("could not read link, skipping") } - - n, err := node.ReadNode(ctx, fs.lu, spaceID, filepath.Base(target)) + nodeID := strings.TrimLeft(target, "/.") + nodeID = strings.ReplaceAll(nodeID, "/", "") + n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID) if err != nil { return nil, err } @@ -370,7 +402,7 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up } } - matches, err := filepath.Glob(filepath.Join(fs.o.Root, "spaces", spaceTypeAny, spaceID)) + matches, err := filepath.Glob(filepath.Join(fs.o.Root, "spacetypes", spaceTypeAny, spaceID)) if err != nil { return nil, err } @@ -388,8 +420,10 @@ func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.Up if err != nil { appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[0]).Msg("could not read link, skipping") } + nodeID := strings.TrimLeft(target, "/.") + nodeID = strings.ReplaceAll(nodeID, "/", "") + node, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID) - node, err := node.ReadNode(ctx, fs.lu, spaceID, filepath.Base(target)) if err != nil { return nil, err } @@ -469,21 +503,7 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De spaceID := req.Id.OpaqueId - matches, err := filepath.Glob(filepath.Join(fs.o.Root, "spaces", spaceTypeAny, spaceID)) - if err != nil { - return err - } - - if len(matches) != 1 { - return fmt.Errorf("delete space failed: found %d matching spaces", len(matches)) - } - - target, err := os.Readlink(matches[0]) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("match", matches[0]).Msg("could not read link, skipping") - } - - n, err := node.ReadNode(ctx, fs.lu, spaceID, filepath.Base(target)) + n, err := node.ReadNode(ctx, fs.lu, spaceID, spaceID) if err != nil { return err } @@ -503,7 +523,7 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De return err } - matches, err = filepath.Glob(filepath.Join(fs.o.Root, "spaces", spaceTypeAny, req.Id.OpaqueId)) + matches, err = filepath.Glob(filepath.Join(fs.o.Root, "spacetypes", spaceTypeAny, req.Id.OpaqueId)) if err != nil { return err } @@ -515,7 +535,7 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De return err } - matches, err = filepath.Glob(filepath.Join(fs.o.Root, "nodes", node.RootID, req.Id.OpaqueId+node.TrashIDDelimiter+"*")) + matches, err = filepath.Glob(filepath.Join(fs.o.Root, "spaces", spaceID, "nodes", node.RootID, req.Id.OpaqueId+node.TrashIDDelimiter+"*")) if err != nil { return err } @@ -535,6 +555,14 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De if err != nil { return err } + matches, err := filepath.Glob(filepath.Join(fs.o.Root, "spacetypes", spaceTypeAny, spaceID)) + if err != nil { + return err + } + + if len(matches) != 1 { + return fmt.Errorf("delete space failed: found %d matching spaces", len(matches)) + } err = os.RemoveAll(matches[0]) if err != nil { @@ -548,12 +576,12 @@ func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.De func (fs *Decomposedfs) createStorageSpace(ctx context.Context, spaceType string, spaceID string) error { // create space type dir - if err := os.MkdirAll(filepath.Join(fs.o.Root, "spaces", spaceType), 0700); err != nil { + if err := os.MkdirAll(filepath.Join(fs.o.Root, "spacetypes", spaceType), 0700); err != nil { return err } // we can reuse the node id as the space id - err := os.Symlink("../../nodes/"+spaceID, filepath.Join(fs.o.Root, "spaces", spaceType, spaceID)) + err := os.Symlink("../../spaces/"+Pathify(spaceID, 1, 2)+"/nodes/"+Pathify(spaceID, 4, 2), filepath.Join(fs.o.Root, "spacetypes", spaceType, spaceID)) if err != nil { if isAlreadyExists(err) { appctx.GetLogger(ctx).Debug().Err(err).Str("space", spaceID).Str("spacetype", spaceType).Msg("symlink already exists") @@ -605,7 +633,7 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, return nil, err } - glob := filepath.Join(fs.o.Root, "spaces", spaceType, n.SpaceRoot.ID) + glob := filepath.Join(fs.o.Root, "spacetypes", spaceType, n.SpaceRoot.ID) matches, err := filepath.Glob(glob) if err != nil { return nil, err diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index acbccb126cb..601bcc9ae2f 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -38,7 +38,6 @@ import ( "github.com/cs3org/reva/pkg/utils" "github.com/google/uuid" "github.com/pkg/errors" - "github.com/pkg/xattr" "github.com/rs/zerolog/log" ) @@ -100,8 +99,8 @@ func (t *Tree) Setup(owner *userpb.UserId, propagateToRoot bool) error { //filepath.Join(t.root, "nodes"), // notes contain symlinks from nodes//uploads/ to ../../uploads/ // better to keep uploads on a fast / volatile storage before a workflow finally moves them to the nodes dir - //filepath.Join(t.root, "uploads"), - //filepath.Join(t.root, "trash"), + filepath.Join(t.root, "uploads"), + filepath.Join(t.root, "trash"), } for _, v := range dataPaths { err := os.MkdirAll(v, 0700) @@ -110,37 +109,12 @@ func (t *Tree) Setup(owner *userpb.UserId, propagateToRoot bool) error { } } - // the root node has an empty name - // the root node has no parent - n := node.New(node.NoSpaceID, node.RootID, "", "", 0, "", nil, t.lookup) - err := t.createNode(n, owner) - if err != nil { - return err - } - - // set propagation flag - v := "0" - if propagateToRoot { - v = "1" - } - if err = n.SetMetadata(xattrs.PropagationAttr, v); err != nil { - return err - } - // create spaces folder and iterate over existing nodes to populate it - spacesPath := filepath.Join(t.root, "spaces") - fi, err := os.Stat(spacesPath) - if os.IsNotExist(err) { - // create personal spaces dir - if err := os.MkdirAll(filepath.Join(spacesPath, spaceTypePersonal), 0700); err != nil { - return err - } - // create share spaces dir - if err := os.MkdirAll(filepath.Join(spacesPath, spaceTypeShare), 0700); err != nil { - return err - } + nodesPath := filepath.Join(t.root, "nodes") + fi, err := os.Stat(nodesPath) + if err == nil && fi.IsDir() { - f, err := os.Open(filepath.Join(t.root, "nodes")) + f, err := os.Open(nodesPath) if err != nil { return err } @@ -149,41 +123,80 @@ func (t *Tree) Setup(owner *userpb.UserId, propagateToRoot bool) error { return err } - for i := range nodes { - nodePath := filepath.Join(t.root, "nodes", nodes[i].Name()) + for _, node := range nodes { + nodePath := filepath.Join(nodesPath, node.Name()) - // is it a user root? -> create personal space if isRootNode(nodePath) { - // we can reuse the node id as the space id - t.linkSpace(spaceTypePersonal, nodes[i].Name(), nodes[i].Name()) + if err := t.moveNode(node.Name(), node.Name()); err != nil { + logger.New().Error().Err(err). + Str("space", node.Name()). + Msg("could not move space") + continue + } + t.linkSpace("personal", node.Name()) } + } + // TODO delete nodesPath if empty + + } - // is it a shared node? -> create share space - if isSharedNode(nodePath) { - // we can reuse the node id as the space id - t.linkSpace(spaceTypeShare, nodes[i].Name(), nodes[i].Name()) + return nil +} +func (t *Tree) moveNode(spaceID, nodeID string) error { + dirPath := filepath.Join(t.root, "nodes", nodeID) + f, err := os.Open(dirPath) + if err != nil { + return err + } + children, err := f.Readdir(0) + if err != nil { + return err + } + for _, child := range children { + old := filepath.Join(t.root, "nodes", child.Name()) + new := filepath.Join(t.root, "spaces", spaceID, "nodes", child.Name()) + if err := os.Rename(old, new); err != nil { + logger.New().Error().Err(err). + Str("space", spaceID). + Str("nodes", child.Name()). + Str("oldpath", old). + Str("newpath", new). + Msg("could not rename node") + } + if child.IsDir() { + if err := t.moveNode(spaceID, child.Name()); err != nil { + return err } } - } else if !fi.IsDir() { - // check if it is a directory - return fmt.Errorf("%s is not a directory", spacesPath) } - return nil } +func Pathify(id string, depth, width int) string { + b := strings.Builder{} + i := 0 + for ; i < depth; i++ { + if len(id) <= i*width+width { + break + } + b.WriteString(id[i*width : i*width+width]) + b.WriteRune(filepath.Separator) + } + b.WriteString(id[i*width:]) + return b.String() +} + // linkSpace creates a new symbolic link for a space with the given type st, and node id -func (t *Tree) linkSpace(spaceType, spaceID, nodeID string) { - spacesPath := filepath.Join(t.root, "spaces", spaceType, spaceID) - expectedTarget := "../../nodes/" + spaceID + nodeID - linkTarget, err := os.Readlink(spacesPath) +func (t *Tree) linkSpace(spaceType, spaceID string) { + spaceTypesPath := filepath.Join(t.root, "spacetypes", spaceType, spaceID) + expectedTarget := "../../spaces/" + Pathify(spaceID, 1, 2) + "/nodes/" + Pathify(spaceID, 4, 2) + linkTarget, err := os.Readlink(spaceTypesPath) if errors.Is(err, os.ErrNotExist) { - err = os.Symlink(expectedTarget, spacesPath) + err = os.Symlink(expectedTarget, spaceTypesPath) if err != nil { logger.New().Error().Err(err). Str("space_type", spaceType). Str("space", spaceID). - Str("node", nodeID). Msg("could not create symlink") } } else { @@ -191,14 +204,12 @@ func (t *Tree) linkSpace(spaceType, spaceID, nodeID string) { logger.New().Error().Err(err). Str("space_type", spaceType). Str("space", spaceID). - Str("node", nodeID). Msg("could not read symlink") } if linkTarget != expectedTarget { logger.New().Warn(). Str("space_type", spaceType). Str("space", spaceID). - Str("node", nodeID). Str("expected", expectedTarget). Str("actual", linkTarget). Msg("expected a different link target") @@ -210,6 +221,8 @@ func isRootNode(nodePath string) bool { attr, err := xattrs.Get(nodePath, xattrs.ParentidAttr) return err == nil && attr == node.RootID } + +/* func isSharedNode(nodePath string) bool { if attrs, err := xattr.List(nodePath); err == nil { for i := range attrs { @@ -220,6 +233,7 @@ func isSharedNode(nodePath string) bool { } return false } +*/ // GetMD returns the metadata of a node in the tree func (t *Tree) GetMD(ctx context.Context, n *node.Node) (os.FileInfo, error) { @@ -264,7 +278,8 @@ func (t *Tree) CreateDir(ctx context.Context, n *node.Node) (err error) { } // make child appear in listings - err = os.Symlink(n.ID, filepath.Join(n.ParentInternalPath(), n.Name)) + relativeNodePath := filepath.Join("../../../../../", Pathify(n.ID, 4, 2)) + err = os.Symlink(relativeNodePath, filepath.Join(n.ParentInternalPath(), n.Name)) if err != nil { // no better way to check unfortunately if !strings.Contains(err.Error(), "file exists") { @@ -354,6 +369,16 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) return nil } +func readChildNodeFromLink(path string) (string, error) { + link, err := os.Readlink(path) + if err != nil { + return "", err + } + nodeID := strings.TrimLeft(link, "/.") + nodeID = strings.ReplaceAll(nodeID, "/", "") + return nodeID, nil +} + // ListFolder lists the content of a folder node func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, error) { dir := n.InternalPath() @@ -372,13 +397,13 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro } nodes := []*node.Node{} for i := range names { - link, err := os.Readlink(filepath.Join(dir, names[i])) + nodeID, err := readChildNodeFromLink(filepath.Join(dir, names[i])) if err != nil { // TODO log continue } - child, err := node.ReadNode(ctx, t.lookup, n.SpaceID, filepath.Base(link)) + child, err := node.ReadNode(ctx, t.lookup, n.SpaceID, nodeID) if err != nil { // TODO log continue diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 3fcebc13f33..aefc953811a 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -467,12 +467,7 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { nil, upload.fs.lu, ) - n.SpaceRoot = node.New(node.NoSpaceID, spaceID, "", "", 0, "", nil, upload.fs.lu) - - // check lock - if err := n.CheckLock(ctx); err != nil { - return err - } + n.SpaceRoot = node.New(spaceID, spaceID, "", "", 0, "", nil, upload.fs.lu) // check lock if err := n.CheckLock(ctx); err != nil { @@ -580,6 +575,7 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { Msg("Decomposedfs: could not truncate") return } + os.MkdirAll(filepath.Dir(targetPath), 0700) if err = os.Rename(upload.binPath, targetPath); err != nil { sublog.Err(err). Msg("Decomposedfs: could not rename") @@ -633,7 +629,8 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { } } if os.IsNotExist(err) || link != "../"+n.ID { - if err = os.Symlink("../"+n.ID, childNameLink); err != nil { + relativeNodePath := filepath.Join("../../../../../", Pathify(n.ID, 4, 2)) + if err = os.Symlink(relativeNodePath, childNameLink); err != nil { return errors.Wrap(err, "Decomposedfs: could not symlink child entry") } }