diff --git a/changelog/unreleased/consolidate-xattr.md b/changelog/unreleased/consolidate-xattr.md new file mode 100644 index 00000000000..845fe7c1353 --- /dev/null +++ b/changelog/unreleased/consolidate-xattr.md @@ -0,0 +1,7 @@ +Enhancement: Consolidate xattr setter and getter + +- Consolidate all metadata Get's and Set's to central functions. +- Cleaner code by reduction of casts +- Easier to hook functionality like indexing + +https://github.com/cs3org/reva/pull/2512 diff --git a/pkg/storage/utils/decomposedfs/decomposedfs.go b/pkg/storage/utils/decomposedfs/decomposedfs.go index 002baf8ea8a..4dd5b313f79 100644 --- a/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -50,7 +50,6 @@ import ( rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" "github.com/pkg/errors" - "github.com/pkg/xattr" "go.opentelemetry.io/otel/codes" ) @@ -234,14 +233,13 @@ func (fs *Decomposedfs) CreateHome(ctx context.Context) (err error) { // update the owner u := ctxpkg.ContextMustGetUser(ctx) - if err = h.WriteMetadata(u.Id); err != nil { + if err = h.WriteAllNodeMetadata(u.Id); err != nil { return } if fs.o.TreeTimeAccounting || fs.o.TreeSizeAccounting { - homePath := h.InternalPath() // mark the home node as the end of propagation - if err = xattr.Set(homePath, xattrs.PropagationAttr, []byte("1")); err != nil { + if err = h.SetMetadata(xattrs.PropagationAttr, "1"); err != nil { appctx.GetLogger(ctx).Error().Err(err).Interface("node", h).Msg("could not mark home as propagation root") return } @@ -340,9 +338,8 @@ func (fs *Decomposedfs) CreateDir(ctx context.Context, ref *provider.Reference) } if fs.o.TreeTimeAccounting || fs.o.TreeSizeAccounting { - nodePath := n.InternalPath() // mark the home node as the end of propagation - if err = xattr.Set(nodePath, xattrs.PropagationAttr, []byte("1")); err != nil { + if err = n.SetMetadata(xattrs.PropagationAttr, "1"); err != nil { appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not mark node to propagate") // FIXME: This does not return an error at all, but results in a severe situation that the @@ -433,12 +430,11 @@ func (fs *Decomposedfs) CreateReference(ctx context.Context, p string, targetURI } childCreated = true - internalPath := childNode.InternalPath() - if err := xattr.Set(internalPath, xattrs.ReferenceAttr, []byte(targetURI.String())); err != nil { + if err := childNode.SetMetadata(xattrs.ReferenceAttr, targetURI.String()); err != nil { // the reference could not be set - that would result in an lost reference? err := errors.Wrapf(err, "Decomposedfs: error setting the target %s on the reference file %s", targetURI.String(), - internalPath, + childNode.InternalPath(), ) span.SetStatus(codes.Error, err.Error()) return err diff --git a/pkg/storage/utils/decomposedfs/grants.go b/pkg/storage/utils/decomposedfs/grants.go index c7404d008da..c41cb37f92d 100644 --- a/pkg/storage/utils/decomposedfs/grants.go +++ b/pkg/storage/utils/decomposedfs/grants.go @@ -62,9 +62,14 @@ func (fs *Decomposedfs) AddGrant(ctx context.Context, ref *provider.Reference, g return errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name)) } + // check lock + if err := node.CheckLock(ctx); err != nil { + return err + } + e := ace.FromGrant(g) principal, value := e.Marshal() - if err := xattr.Set(node.InternalPath(), xattrs.GrantPrefix+principal, value); err != nil { + if err := node.SetMetadata(xattrs.GrantPrefix+principal, string(value)); err != nil { return err } @@ -172,15 +177,15 @@ func extractACEsFromAttrs(ctx context.Context, fsfn string, attrs []string) (ent entries = []*ace.ACE{} for i := range attrs { if strings.HasPrefix(attrs[i], xattrs.GrantPrefix) { - var value []byte + var value string var err error - if value, err = xattr.Get(fsfn, attrs[i]); err != nil { + if value, err = xattrs.Get(fsfn, attrs[i]); err != nil { log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute") continue } var e *ace.ACE principal := attrs[i][len(xattrs.GrantPrefix):] - if e, err = ace.Unmarshal(principal, value); err != nil { + if e, err = ace.Unmarshal(principal, []byte(value)); err != nil { log.Error().Err(err).Str("principal", principal).Str("attr", attrs[i]).Msg("could not unmarshal ace") continue } diff --git a/pkg/storage/utils/decomposedfs/lookup.go b/pkg/storage/utils/decomposedfs/lookup.go index 05f0973d827..84e79095f23 100644 --- a/pkg/storage/utils/decomposedfs/lookup.go +++ b/pkg/storage/utils/decomposedfs/lookup.go @@ -33,7 +33,6 @@ import ( "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/options" "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" "github.com/cs3org/reva/pkg/storage/utils/templates" - "github.com/pkg/xattr" ) // Lookup implements transformations from filepath to node and back @@ -157,9 +156,9 @@ func (lu *Lookup) WalkPath(ctx context.Context, r *node.Node, p string, followRe } if followReferences { - if attrBytes, err := xattr.Get(r.InternalPath(), xattrs.ReferenceAttr); err == nil { + if attrBytes, err := r.GetMetadata(xattrs.ReferenceAttr); err == nil { realNodeID := attrBytes - ref, err := xattrs.ReferenceFromAttr(realNodeID) + ref, err := xattrs.ReferenceFromAttr([]byte(realNodeID)) if err != nil { return nil, err } diff --git a/pkg/storage/utils/decomposedfs/node/node.go b/pkg/storage/utils/decomposedfs/node/node.go index 0794a217079..1e8253aaeea 100644 --- a/pkg/storage/utils/decomposedfs/node/node.go +++ b/pkg/storage/utils/decomposedfs/node/node.go @@ -113,14 +113,13 @@ func New(spaceID, id, parentID, name string, blobsize int64, blobID string, owne func (n *Node) ChangeOwner(new *userpb.UserId) (err error) { nodePath := n.InternalPath() n.owner = new - if err = xattr.Set(nodePath, xattrs.OwnerIDAttr, []byte(new.OpaqueId)); err != nil { - return errors.Wrap(err, "Decomposedfs: could not reset owner id attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerIDPAttr, []byte(new.Idp)); err != nil { - return errors.Wrap(err, "Decomposedfs: could not reset owner idp attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerTypeAttr, []byte(utils.UserTypeToString(new.Type))); err != nil { - return errors.Wrap(err, "Decomposedfs: could not reset owner idp attribute") + + var attribs = map[string]string{xattrs.OwnerIDAttr: new.OpaqueId, + xattrs.OwnerIDPAttr: new.Idp, + xattrs.OwnerTypeAttr: utils.UserTypeToString(new.Type)} + + if err := xattrs.SetMultiple(nodePath, attribs); err != nil { + return err } return @@ -130,47 +129,42 @@ func (n *Node) ChangeOwner(new *userpb.UserId) (err error) { // Note that consumers should be aware of the metadata options on xattrs.go. func (n *Node) SetMetadata(key string, val string) (err error) { nodePath := n.InternalPath() - if err := xattr.Set(nodePath, key, []byte(val)); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set parentid attribute") + if err := xattrs.Set(nodePath, key, val); err != nil { + return errors.Wrap(err, "Decomposedfs: could not set extended attribute") } return nil } -// WriteMetadata writes the Node metadata to disk -func (n *Node) WriteMetadata(owner *userpb.UserId) (err error) { +// GetMetadata reads the metadata for the given key +func (n *Node) GetMetadata(key string) (val string, err error) { nodePath := n.InternalPath() - if err = xattr.Set(nodePath, xattrs.ParentidAttr, []byte(n.ParentID)); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set parentid attribute") - } - if err = xattr.Set(nodePath, xattrs.NameAttr, []byte(n.Name)); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set name attribute") - } - if err = xattr.Set(nodePath, xattrs.BlobIDAttr, []byte(n.BlobID)); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set blobid attribute") + if val, err = xattrs.Get(nodePath, key); err != nil { + return "", errors.Wrap(err, "Decomposedfs: could not get extended attribute") } - if err = xattr.Set(nodePath, xattrs.BlobsizeAttr, []byte(fmt.Sprintf("%d", n.Blobsize))); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set blobsize attribute") + return val, nil +} + +// WriteAllNodeMetadata writes the Node metadata to disk +func (n *Node) WriteAllNodeMetadata(owner *userpb.UserId) (err error) { + attribs := make(map[string]string) + + attribs[xattrs.ParentidAttr] = n.ParentID + attribs[xattrs.NameAttr] = n.Name + attribs[xattrs.BlobIDAttr] = n.BlobID + attribs[xattrs.BlobsizeAttr] = strconv.FormatInt(n.Blobsize, 10) + + nodePath := n.InternalPath() + attribs[xattrs.OwnerIDAttr] = "" + attribs[xattrs.OwnerIDPAttr] = "" + attribs[xattrs.OwnerTypeAttr] = "" + + if owner != nil { + attribs[xattrs.OwnerIDAttr] = owner.OpaqueId + attribs[xattrs.OwnerIDPAttr] = owner.Idp + attribs[xattrs.OwnerTypeAttr] = utils.UserTypeToString(owner.Type) } - if owner == nil { - if err = xattr.Set(nodePath, xattrs.OwnerIDAttr, []byte("")); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set empty owner id attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerIDPAttr, []byte("")); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set empty owner idp attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerTypeAttr, []byte("")); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set empty owner type attribute") - } - } else { - if err = xattr.Set(nodePath, xattrs.OwnerIDAttr, []byte(owner.OpaqueId)); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set owner id attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerIDPAttr, []byte(owner.Idp)); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set owner idp attribute") - } - if err = xattr.Set(nodePath, xattrs.OwnerTypeAttr, []byte(utils.UserTypeToString(owner.Type))); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set owner idp attribute") - } + if err := xattrs.SetMultiple(nodePath, attribs); err != nil { + return err } return } @@ -186,11 +180,11 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string) (n *No nodePath := n.InternalPath() // lookup parent id in extended attributes - var attrBytes []byte - attrBytes, err = xattr.Get(nodePath, xattrs.ParentidAttr) + var attr string + attr, err = xattrs.Get(nodePath, xattrs.ParentidAttr) switch { case err == nil: - n.ParentID = string(attrBytes) + n.ParentID = attr case isAttrUnset(err): return nil, errtypes.InternalError(err.Error()) case isNotFound(err): @@ -200,18 +194,18 @@ func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string) (n *No } // check if this is a space root - if _, err = xattr.Get(nodePath, xattrs.SpaceNameAttr); err == nil { + if _, err = xattrs.Get(nodePath, xattrs.SpaceNameAttr); err == nil { n.SpaceRoot = n } // lookup name in extended attributes - if attrBytes, err = xattr.Get(nodePath, xattrs.NameAttr); err == nil { - n.Name = string(attrBytes) + if attr, err = xattrs.Get(nodePath, xattrs.NameAttr); err == nil { + n.Name = attr } else { return } // lookup blobID in extended attributes - if attrBytes, err = xattr.Get(nodePath, xattrs.BlobIDAttr); err == nil { - n.BlobID = string(attrBytes) + if attr, err = xattrs.Get(nodePath, xattrs.BlobIDAttr); err == nil { + n.BlobID = attr } else { return } @@ -301,16 +295,14 @@ func (n *Node) Parent() (p *Node, err error) { parentPath := p.InternalPath() // lookup parent id in extended attributes - var attrBytes []byte - if attrBytes, err = xattr.Get(parentPath, xattrs.ParentidAttr); err == nil { - p.ParentID = string(attrBytes) - } else { + if p.ParentID, err = xattrs.Get(parentPath, xattrs.ParentidAttr); err != nil { + p.ParentID = "" return } // lookup name in extended attributes - if attrBytes, err = xattr.Get(parentPath, xattrs.NameAttr); err == nil { - p.Name = string(attrBytes) - } else { + if p.Name, err = xattrs.Get(parentPath, xattrs.NameAttr); err != nil { + p.Name = "" + p.ParentID = "" return } @@ -336,13 +328,13 @@ func (n *Node) Owner() (*userpb.UserId, error) { // TODO what if this is a reference? nodePath := n.InternalPath() // lookup parent id in extended attributes - var attrBytes []byte + var attr string var err error // lookup ID in extended attributes - attrBytes, err = xattr.Get(nodePath, xattrs.OwnerIDAttr) + attr, err = xattrs.Get(nodePath, xattrs.OwnerIDAttr) switch { case err == nil: - owner.OpaqueId = string(attrBytes) + owner.OpaqueId = attr case isAttrUnset(err), isNotFound(err): fallthrough default: @@ -350,10 +342,10 @@ func (n *Node) Owner() (*userpb.UserId, error) { } // lookup IDP in extended attributes - attrBytes, err = xattr.Get(nodePath, xattrs.OwnerIDPAttr) + attr, err = xattrs.Get(nodePath, xattrs.OwnerIDPAttr) switch { case err == nil: - owner.Idp = string(attrBytes) + owner.Idp = attr case isAttrUnset(err), isNotFound(err): fallthrough default: @@ -361,10 +353,10 @@ func (n *Node) Owner() (*userpb.UserId, error) { } // lookup type in extended attributes - attrBytes, err = xattr.Get(nodePath, xattrs.OwnerTypeAttr) + attr, err = xattrs.Get(nodePath, xattrs.OwnerTypeAttr) switch { case err == nil: - owner.Type = utils.UserTypeMap(string(attrBytes)) + owner.Type = utils.UserTypeMap(attr) case isAttrUnset(err), isNotFound(err): fallthrough default: @@ -482,7 +474,7 @@ func (n *Node) SetEtag(ctx context.Context, val string) (err error) { return nil } // etag is only valid until the calculated etag changes, is part of propagation - return xattr.Set(nodePath, xattrs.TmpEtagAttr, []byte(val)) + return xattrs.Set(nodePath, xattrs.TmpEtagAttr, val) } // SetFavorite sets the favorite for the current user @@ -506,7 +498,7 @@ func (n *Node) SetFavorite(uid *userpb.UserId, val string) error { nodePath := n.InternalPath() // the favorite flag is specific to the user, so we need to incorporate the userid fa := fmt.Sprintf("%s:%s:%s@%s", xattrs.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) - return xattr.Set(nodePath, fa, []byte(val)) + return xattrs.Set(nodePath, fa, val) } // AsResourceInfo return the node as CS3 ResourceInfo @@ -523,10 +515,10 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi return } - var target []byte + var target string switch { case fi.IsDir(): - if target, err = xattr.Get(nodePath, xattrs.ReferenceAttr); err == nil { + if target, err = xattrs.Get(nodePath, xattrs.ReferenceAttr); err == nil { nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE } else { nodeType = provider.ResourceType_RESOURCE_TYPE_CONTAINER @@ -557,7 +549,7 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi Type: nodeType, MimeType: mime.Detect(nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER, fn), Size: uint64(n.Blobsize), - Target: string(target), + Target: target, PermissionSet: rp, } @@ -584,8 +576,8 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi } // use temporary etag if it is set - if b, err := xattr.Get(nodePath, xattrs.TmpEtagAttr); err == nil { - ri.Etag = fmt.Sprintf(`"%x"`, string(b)) // TODO why do we convert string(b)? is the temporary etag stored as string? -> should we use bytes? use hex.EncodeToString? + if b, err := xattrs.Get(nodePath, xattrs.TmpEtagAttr); err == nil { + ri.Etag = fmt.Sprintf(`"%x"`, b) // TODO why do we convert string(b)? is the temporary etag stored as string? -> should we use bytes? use hex.EncodeToString? } else if ri.Etag, err = calculateEtag(n.ID, tmTime); err != nil { sublog.Debug().Err(err).Msg("could not calculate etag") } @@ -617,11 +609,11 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi // the favorite flag is specific to the user, so we need to incorporate the userid if uid := u.GetId(); uid != nil { fa := fmt.Sprintf("%s:%s:%s@%s", xattrs.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) - if val, err := xattr.Get(nodePath, fa); err == nil { + if val, err := xattrs.Get(nodePath, fa); err == nil { sublog.Debug(). Str("favorite", fa). Msg("found favorite flag") - favorite = string(val) + favorite = val } } else { sublog.Error().Err(errtypes.UserRequired("userrequired")).Msg("user has no id") @@ -683,8 +675,8 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi // only read when key was requested k := attrs[i][len(xattrs.MetadataPrefix):] if _, ok := mdKeysMap[k]; returnAllKeys || ok { - if val, err := xattr.Get(nodePath, attrs[i]); err == nil { - metadata[k] = string(val) + if val, err := xattrs.Get(nodePath, attrs[i]); err == nil { + metadata[k] = val } else { sublog.Error().Err(err). Str("entry", attrs[i]). @@ -706,12 +698,12 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi } func readChecksumIntoResourceChecksum(ctx context.Context, nodePath, algo string, ri *provider.ResourceInfo) { - v, err := xattr.Get(nodePath, xattrs.ChecksumPrefix+algo) + v, err := xattrs.Get(nodePath, xattrs.ChecksumPrefix+algo) switch { case err == nil: ri.Checksum = &provider.ResourceChecksum{ Type: storageprovider.PKG2GRPCXS(algo), - Sum: hex.EncodeToString(v), + Sum: hex.EncodeToString([]byte(v)), } case isAttrUnset(err): appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", nodePath).Str("algorithm", algo).Msg("checksum not set") @@ -723,7 +715,7 @@ func readChecksumIntoResourceChecksum(ctx context.Context, nodePath, algo string } func readChecksumIntoOpaque(ctx context.Context, nodePath, algo string, ri *provider.ResourceInfo) { - v, err := xattr.Get(nodePath, xattrs.ChecksumPrefix+algo) + v, err := xattrs.Get(nodePath, xattrs.ChecksumPrefix+algo) switch { case err == nil: if ri.Opaque == nil { @@ -733,7 +725,7 @@ func readChecksumIntoOpaque(ctx context.Context, nodePath, algo string, ri *prov } ri.Opaque.Map[algo] = &types.OpaqueEntry{ Decoder: "plain", - Value: []byte(hex.EncodeToString(v)), + Value: []byte(hex.EncodeToString([]byte(v))), } case isAttrUnset(err): appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", nodePath).Str("algorithm", algo).Msg("checksum not set") @@ -746,7 +738,7 @@ func readChecksumIntoOpaque(ctx context.Context, nodePath, algo string, ri *prov // quota is always stored on the root node func readQuotaIntoOpaque(ctx context.Context, nodePath string, ri *provider.ResourceInfo) { - v, err := xattr.Get(nodePath, xattrs.QuotaAttr) + v, err := xattrs.Get(nodePath, xattrs.QuotaAttr) switch { case err == nil: // make sure we have a proper signed int @@ -754,7 +746,7 @@ func readQuotaIntoOpaque(ctx context.Context, nodePath string, ri *provider.Reso // -1 = uncalculated // -2 = unknown // -3 = unlimited - if _, err := strconv.ParseInt(string(v), 10, 64); err == nil { + if _, err := strconv.ParseInt(v, 10, 64); err == nil { if ri.Opaque == nil { ri.Opaque = &types.Opaque{ Map: map[string]*types.OpaqueEntry{}, @@ -762,10 +754,10 @@ func readQuotaIntoOpaque(ctx context.Context, nodePath string, ri *provider.Reso } ri.Opaque.Map[QuotaKey] = &types.OpaqueEntry{ Decoder: "plain", - Value: v, + Value: []byte(v), } } else { - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("quota", string(v)).Msg("malformed quota") + appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", nodePath).Str("quota", v).Msg("malformed quota") } case isAttrUnset(err): appctx.GetLogger(ctx).Debug().Err(err).Str("nodepath", nodePath).Msg("quota not set") @@ -778,43 +770,43 @@ func readQuotaIntoOpaque(ctx context.Context, nodePath string, ri *provider.Reso // HasPropagation checks if the propagation attribute exists and is set to "1" func (n *Node) HasPropagation() (propagation bool) { - if b, err := xattr.Get(n.InternalPath(), xattrs.PropagationAttr); err == nil { - return string(b) == "1" + if b, err := xattrs.Get(n.InternalPath(), xattrs.PropagationAttr); err == nil { + return b == "1" } return false } // GetTMTime reads the tmtime from the extended attributes func (n *Node) GetTMTime() (tmTime time.Time, err error) { - var b []byte - if b, err = xattr.Get(n.InternalPath(), xattrs.TreeMTimeAttr); err != nil { + var b string + if b, err = xattrs.Get(n.InternalPath(), xattrs.TreeMTimeAttr); err != nil { return } - return time.Parse(time.RFC3339Nano, string(b)) + return time.Parse(time.RFC3339Nano, b) } // SetTMTime writes the tmtime to the extended attributes func (n *Node) SetTMTime(t time.Time) (err error) { - return xattr.Set(n.InternalPath(), xattrs.TreeMTimeAttr, []byte(t.UTC().Format(time.RFC3339Nano))) + return xattrs.Set(n.InternalPath(), xattrs.TreeMTimeAttr, t.UTC().Format(time.RFC3339Nano)) } // GetTreeSize reads the treesize from the extended attributes func (n *Node) GetTreeSize() (treesize uint64, err error) { - var b []byte - if b, err = xattr.Get(n.InternalPath(), xattrs.TreesizeAttr); err != nil { + var b string + if b, err = xattrs.Get(n.InternalPath(), xattrs.TreesizeAttr); err != nil { return } - return strconv.ParseUint(string(b), 10, 64) + return strconv.ParseUint(b, 10, 64) } // SetTreeSize writes the treesize to the extended attributes func (n *Node) SetTreeSize(ts uint64) (err error) { - return xattr.Set(n.InternalPath(), xattrs.TreesizeAttr, []byte(strconv.FormatUint(ts, 10))) + return n.SetMetadata(xattrs.TreesizeAttr, strconv.FormatUint(ts, 10)) } // SetChecksum writes the checksum with the given checksum type to the extended attributes func (n *Node) SetChecksum(csType string, h hash.Hash) (err error) { - return xattr.Set(n.InternalPath(), xattrs.ChecksumPrefix+csType, h.Sum(nil)) + return n.SetMetadata(xattrs.ChecksumPrefix+csType, string(h.Sum(nil))) } // UnsetTempEtag removes the temporary etag attribute @@ -913,6 +905,7 @@ func (n *Node) ReadUserPermissions(ctx context.Context, u *userpb.User) (ap prov // The function will return a list of opaque strings that can be used to make a ReadGrant call func (n *Node) ListGrantees(ctx context.Context) (grantees []string, err error) { var attrs []string + if attrs, err = xattr.List(n.InternalPath()); err != nil { appctx.GetLogger(ctx).Error().Err(err).Str("node", n.ID).Msg("error listing attributes") return nil, err @@ -927,12 +920,12 @@ func (n *Node) ListGrantees(ctx context.Context) (grantees []string, err error) // ReadGrant reads a CS3 grant func (n *Node) ReadGrant(ctx context.Context, grantee string) (g *provider.Grant, err error) { - var b []byte - if b, err = xattr.Get(n.InternalPath(), grantee); err != nil { + var b string + if b, err = xattrs.Get(n.InternalPath(), grantee); err != nil { return nil, err } var e *ace.ACE - if e, err = ace.Unmarshal(strings.TrimPrefix(grantee, xattrs.GrantPrefix), b); err != nil { + if e, err = ace.Unmarshal(strings.TrimPrefix(grantee, xattrs.GrantPrefix), []byte(b)); err != nil { return nil, err } return e.Grant(), nil @@ -964,11 +957,11 @@ func (n *Node) ListGrants(ctx context.Context) ([]*provider.Grant, error) { // ReadBlobSizeAttr reads the blobsize from the xattrs func ReadBlobSizeAttr(path string) (int64, error) { - attrBytes, err := xattr.Get(path, xattrs.BlobsizeAttr) + attr, err := xattrs.Get(path, xattrs.BlobsizeAttr) if err != nil { return 0, errors.Wrapf(err, "error reading blobsize xattr") } - blobSize, err := strconv.ParseInt(string(attrBytes), 10, 64) + blobSize, err := strconv.ParseInt(attr, 10, 64) if err != nil { return 0, errors.Wrapf(err, "invalid blobsize xattr format") } @@ -1025,7 +1018,7 @@ func (n *Node) FindStorageSpaceRoot() error { // IsSpaceRoot checks if the node is a space root func IsSpaceRoot(r *Node) bool { path := r.InternalPath() - if _, err := xattr.Get(path, xattrs.SpaceNameAttr); err == nil { + if _, err := xattrs.Get(path, xattrs.SpaceNameAttr); err == nil { return true } return false @@ -1037,13 +1030,13 @@ var CheckQuota = func(spaceRoot *Node, fileSize uint64) (quotaSufficient bool, e if !enoughDiskSpace(spaceRoot.InternalPath(), fileSize) { return false, errtypes.InsufficientStorage("disk full") } - quotaByte, _ := xattr.Get(spaceRoot.InternalPath(), xattrs.QuotaAttr) + quotaByte, _ := xattrs.Get(spaceRoot.InternalPath(), xattrs.QuotaAttr) var total uint64 - if quotaByte == nil { + if quotaByte == "" { // if quota is not set, it means unlimited return true, nil } - total, _ = strconv.ParseUint(string(quotaByte), 10, 64) + total, _ = strconv.ParseUint(quotaByte, 10, 64) // if total is smaller than used, total-used could overflow and be bigger than fileSize if fileSize > total-used || total < used { return false, errtypes.InsufficientStorage("quota exceeded") diff --git a/pkg/storage/utils/decomposedfs/node/node_test.go b/pkg/storage/utils/decomposedfs/node/node_test.go index 4cf5420cf3f..9c422d11a4a 100644 --- a/pkg/storage/utils/decomposedfs/node/node_test.go +++ b/pkg/storage/utils/decomposedfs/node/node_test.go @@ -97,7 +97,7 @@ var _ = Describe("Node", func() { Type: userpb.UserType_USER_TYPE_PRIMARY, } - err = n.WriteMetadata(owner) + err = n.WriteAllNodeMetadata(owner) Expect(err).ToNot(HaveOccurred()) n2, err := env.Lookup.NodeFromResource(env.Ctx, ref) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/storage/utils/decomposedfs/spaces.go b/pkg/storage/utils/decomposedfs/spaces.go index 867c02ad1c6..2fdde023301 100644 --- a/pkg/storage/utils/decomposedfs/spaces.go +++ b/pkg/storage/utils/decomposedfs/spaces.go @@ -44,7 +44,6 @@ import ( "github.com/cs3org/reva/pkg/storage/utils/decomposedfs/xattrs" "github.com/cs3org/reva/pkg/utils" "github.com/google/uuid" - "github.com/pkg/xattr" ) const ( @@ -92,9 +91,8 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr } // always enable propagation on the storage space root - nodePath := n.InternalPath() // mark the space root node as the end of propagation - if err = xattr.Set(nodePath, xattrs.PropagationAttr, []byte("1")); err != nil { + if err = n.SetMetadata(xattrs.PropagationAttr, "1"); err != nil { appctx.GetLogger(ctx).Error().Err(err).Interface("node", n).Msg("could not mark node to propagate") return nil, err } @@ -549,10 +547,10 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, } // TODO apply more filters - - sname := "" - if bytes, err := xattr.Get(n.InternalPath(), xattrs.SpaceNameAttr); err == nil { - sname = string(bytes) + var sname string + if sname, err = n.GetMetadata(xattrs.SpaceNameAttr); err != nil { + // FIXME: Is that a severe problem? + appctx.GetLogger(ctx).Debug().Err(err).Msg("space does not have a name attribute") } if err := n.FindStorageSpaceRoot(); err != nil { @@ -643,14 +641,14 @@ func (fs *Decomposedfs) storageSpaceFromNode(ctx context.Context, n *node.Node, } // quota - v, err := xattr.Get(nodePath, xattrs.QuotaAttr) + v, err := xattrs.Get(nodePath, xattrs.QuotaAttr) if err == nil { // make sure we have a proper signed int // we use the same magic numbers to indicate: // -1 = uncalculated // -2 = unknown // -3 = unlimited - if quota, err := strconv.ParseUint(string(v), 10, 64); err == nil { + if quota, err := strconv.ParseUint(v, 10, 64); err == nil { space.Quota = &provider.Quota{ QuotaMaxBytes: quota, QuotaMaxFiles: math.MaxUint64, // TODO MaxUInt64? = unlimited? why even max files? 0 = unlimited? diff --git a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go index 3d004c95e27..ed61673368a 100644 --- a/pkg/storage/utils/decomposedfs/testhelpers/helpers.go +++ b/pkg/storage/utils/decomposedfs/testhelpers/helpers.go @@ -152,7 +152,7 @@ func (t *TestEnv) CreateTestFile(name, blobID, parentID, spaceID string, blobSiz if err != nil { return nil, err } - err = n.WriteMetadata(t.Owner.Id) + err = file.WriteAllNodeMetadata(t.Owner.Id) if err != nil { return nil, err } diff --git a/pkg/storage/utils/decomposedfs/tree/tree.go b/pkg/storage/utils/decomposedfs/tree/tree.go index f75d6c8e890..2fdf027f14c 100644 --- a/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/pkg/storage/utils/decomposedfs/tree/tree.go @@ -118,11 +118,11 @@ func (t *Tree) Setup(owner *userpb.UserId, propagateToRoot bool) error { } // set propagation flag - v := []byte("0") + v := "0" if propagateToRoot { - v = []byte("1") + v = "1" } - if err = xattr.Set(n.InternalPath(), xattrs.PropagationAttr, v); err != nil { + if err = n.SetMetadata(xattrs.PropagationAttr, v); err != nil { return err } @@ -206,8 +206,8 @@ func (t *Tree) linkSpace(spaceType, spaceID, nodeID string) { } func isRootNode(nodePath string) bool { - attrBytes, err := xattr.Get(nodePath, xattrs.ParentidAttr) - return err == nil && string(attrBytes) == "root" + attr, err := xattrs.Get(nodePath, xattrs.ParentidAttr) + return err == nil && attr == "root" } func isSharedNode(nodePath string) bool { if attrs, err := xattr.List(nodePath); err == nil { @@ -311,7 +311,7 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) } // update name attribute - if err := xattr.Set(tgtPath, xattrs.NameAttr, []byte(newNode.Name)); err != nil { + if err := xattrs.Set(tgtPath, xattrs.NameAttr, newNode.Name); err != nil { return errors.Wrap(err, "Decomposedfs: could not set name attribute") } @@ -331,10 +331,10 @@ func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) } // update target parentid and name - if err := xattr.Set(tgtPath, xattrs.ParentidAttr, []byte(newNode.ParentID)); err != nil { + if err := xattrs.Set(tgtPath, xattrs.ParentidAttr, newNode.ParentID); err != nil { return errors.Wrap(err, "Decomposedfs: could not set parentid attribute") } - if err := xattr.Set(tgtPath, xattrs.NameAttr, []byte(newNode.Name)); err != nil { + if err := xattrs.Set(tgtPath, xattrs.NameAttr, newNode.Name); err != nil { return errors.Wrap(err, "Decomposedfs: could not set name attribute") } @@ -412,7 +412,7 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { // set origin location in metadata nodePath := n.InternalPath() - if err := xattr.Set(nodePath, xattrs.TrashOriginAttr, []byte(origin)); err != nil { + if err := n.SetMetadata(xattrs.TrashOriginAttr, origin); err != nil { return err } @@ -524,13 +524,13 @@ func (t *Tree) RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPa targetNode.Exists = true // update name attribute - if err := xattr.Set(nodePath, xattrs.NameAttr, []byte(targetNode.Name)); err != nil { + if err := recycleNode.SetMetadata(xattrs.NameAttr, targetNode.Name); err != nil { return errors.Wrap(err, "Decomposedfs: could not set name attribute") } // set ParentidAttr to restorePath's node parent id if trashPath != "" { - if err := xattr.Set(nodePath, xattrs.ParentidAttr, []byte(targetNode.ParentID)); err != nil { + if err := recycleNode.SetMetadata(xattrs.ParentidAttr, targetNode.ParentID); err != nil { return errors.Wrap(err, "Decomposedfs: could not set name attribute") } } @@ -735,13 +735,13 @@ func calculateTreeSize(ctx context.Context, nodePath string) (uint64, error) { size += uint64(blobSize) } else { // read from attr - var b []byte - // xattr.Get will follow the symlink - if b, err = xattr.Get(cPath, xattrs.TreesizeAttr); err != nil { + var b string + // xattrs.Get will follow the symlink + if b, err = xattrs.Get(cPath, xattrs.TreesizeAttr); err != nil { // TODO recursively descend and recalculate treesize continue // continue after an error } - csize, err := strconv.ParseUint(string(b), 10, 64) + csize, err := strconv.ParseUint(b, 10, 64) if err != nil { // TODO recursively descend and recalculate treesize continue // continue after an error @@ -780,7 +780,7 @@ func (t *Tree) createNode(n *node.Node, owner *userpb.UserId) (err error) { return errors.Wrap(err, "Decomposedfs: error creating node") } - return n.WriteMetadata(owner) + return n.WriteAllNodeMetadata(owner) } // TODO refactor the returned params into Node properties? would make all the path transformations go away... @@ -798,48 +798,48 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) ( return } - var attrBytes []byte + var attrStr string trashNodeID := filepath.Base(link) deletedNodePath = t.lookup.InternalPath(spaceID, trashNodeID) owner := &userpb.UserId{} // lookup ownerId in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.OwnerIDAttr); err == nil { - owner.OpaqueId = string(attrBytes) + if attrStr, err = xattrs.Get(deletedNodePath, xattrs.OwnerIDAttr); err == nil { + owner.OpaqueId = attrStr } else { return } // lookup ownerIdp in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.OwnerIDPAttr); err == nil { - owner.Idp = string(attrBytes) + if attrStr, err = xattrs.Get(deletedNodePath, xattrs.OwnerIDPAttr); err == nil { + owner.Idp = attrStr } else { return } // lookup ownerType in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.OwnerTypeAttr); err == nil { - owner.Type = utils.UserTypeMap(string(attrBytes)) + if attrStr, err = xattrs.Get(deletedNodePath, xattrs.OwnerTypeAttr); err == nil { + owner.Type = utils.UserTypeMap(attrStr) } else { return } recycleNode = node.New(spaceID, trashNodeID, "", "", 0, "", owner, t.lookup) // lookup blobID in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.BlobIDAttr); err == nil { - recycleNode.BlobID = string(attrBytes) + if attrStr, err = xattrs.Get(deletedNodePath, xattrs.BlobIDAttr); err == nil { + recycleNode.BlobID = attrStr } else { return } // lookup parent id in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.ParentidAttr); err == nil { - recycleNode.ParentID = string(attrBytes) + if attrStr, err = xattrs.Get(deletedNodePath, xattrs.ParentidAttr); err == nil { + recycleNode.ParentID = attrStr } else { return } // lookup name in extended attributes - if attrBytes, err = xattr.Get(deletedNodePath, xattrs.NameAttr); err == nil { - recycleNode.Name = string(attrBytes) + if attrStr, err = xattrs.Get(deletedNodePath, xattrs.NameAttr); err == nil { + recycleNode.Name = attrStr } else { return } @@ -872,8 +872,8 @@ func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) ( deletedNodeRootPath = t.lookup.InternalPath(spaceID, filepath.Base(rootLink)) } // lookup origin path in extended attributes - if attrBytes, err = xattr.Get(deletedNodeRootPath, xattrs.TrashOriginAttr); err == nil { - origin = filepath.Join(string(attrBytes), path) + if attrStr, err = xattrs.Get(deletedNodeRootPath, xattrs.TrashOriginAttr); err == nil { + origin = filepath.Join(attrStr, path) } else { log.Error().Err(err).Str("trashItem", trashItem).Str("link", link).Str("deletedNodePath", deletedNodePath).Msg("could not read origin path, restoring to /") } diff --git a/pkg/storage/utils/decomposedfs/upload.go b/pkg/storage/utils/decomposedfs/upload.go index 0333f7b96f7..13618d4ac92 100644 --- a/pkg/storage/utils/decomposedfs/upload.go +++ b/pkg/storage/utils/decomposedfs/upload.go @@ -604,7 +604,7 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { tryWritingChecksum(&sublog, n, "adler32", adler32h) // who will become the owner? the owner of the parent actually ... not the currently logged in user - err = n.WriteMetadata(&userpb.UserId{ + err = n.WriteAllNodeMetadata(&userpb.UserId{ Idp: upload.info.Storage["OwnerIdp"], OpaqueId: upload.info.Storage["OwnerId"], }) diff --git a/pkg/storage/utils/decomposedfs/xattrs/xattrs.go b/pkg/storage/utils/decomposedfs/xattrs/xattrs.go index 9688e042fc2..f65900c202a 100644 --- a/pkg/storage/utils/decomposedfs/xattrs/xattrs.go +++ b/pkg/storage/utils/decomposedfs/xattrs/xattrs.go @@ -22,6 +22,7 @@ import ( "strings" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/pkg/errors" "github.com/pkg/xattr" ) @@ -127,3 +128,55 @@ func CopyMetadata(s, t string, filter func(attributeName string) bool) error { } return nil } + +// Set an extended attribute key to the given value +func Set(filePath string, key string, val string) error { + + if err := xattr.Set(filePath, key, []byte(val)); err != nil { + return errors.Wrap(err, "xattrs: Could not write xtended attribute") + } + return nil +} + +// SetMultiple allows setting multiple key value pairs at once +// TODO the changes are protected with an flock +func SetMultiple(filePath string, attribs map[string]string) error { + + // FIXME: Lock here + for key, val := range attribs { + if err := Set(filePath, key, val); err != nil { + return err + } + } + return nil +} + +// Get an extended attribute value for the given key +func Get(filePath, key string) (string, error) { + + v, err := xattr.Get(filePath, key) + if err != nil { + return "", errors.Wrap(err, "xattrs: Can not read xattr") + } + val := string(v) + return val, nil +} + +// All reads all extended attributes for a node +func All(filePath string) (map[string]string, error) { + attrNames, err := xattr.List(filePath) + if err != nil { + return nil, errors.Wrap(err, "xattrs: Can not list extended attributes") + } + + attribs := make(map[string]string, len(attrNames)) + for _, name := range attrNames { + val, err := xattr.Get(filePath, name) + if err != nil { + return nil, errors.Wrap(err, "Failed to read extended attrib") + } + attribs[name] = string(val) + } + + return attribs, nil +}