diff --git a/changelog/unreleased/ocis-driver.md b/changelog/unreleased/ocis-driver.md new file mode 100644 index 00000000000..72f86fc8f47 --- /dev/null +++ b/changelog/unreleased/ocis-driver.md @@ -0,0 +1,5 @@ +Enhancement: Introduce ocis storage driver + +We introduced a now storage driver `ocis` that deconstructs a filesystem and uses a node first approach to implement an efficient lookup of files by path as well as by file id. + +https://github.com/cs3org/reva/pull/559 diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index afec9427c3c..a9984a952b9 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -46,14 +46,13 @@ func init() { type config struct { MountPath string `mapstructure:"mount_path"` - MountID string `mapstructure:"mount_id"` GatewayAddr string `mapstructure:"gateway_addr"` } type service struct { - conf *config - mountPath, mountID string - gateway gateway.GatewayAPIClient + conf *config + mountPath string + gateway gateway.GatewayAPIClient } func (s *service) Close() error { @@ -85,7 +84,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { } mountPath := c.MountPath - mountID := c.MountID gateway, err := pool.GetGatewayServiceClient(c.GatewayAddr) if err != nil { @@ -95,7 +93,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { service := &service{ conf: c, mountPath: mountPath, - mountID: mountID, gateway: gateway, } diff --git a/pkg/storage/fs/loader/loader.go b/pkg/storage/fs/loader/loader.go index 76604757c09..744265bc8ff 100644 --- a/pkg/storage/fs/loader/loader.go +++ b/pkg/storage/fs/loader/loader.go @@ -25,6 +25,7 @@ import ( _ "github.com/cs3org/reva/pkg/storage/fs/eoshome" _ "github.com/cs3org/reva/pkg/storage/fs/local" _ "github.com/cs3org/reva/pkg/storage/fs/localhome" + _ "github.com/cs3org/reva/pkg/storage/fs/ocis" _ "github.com/cs3org/reva/pkg/storage/fs/owncloud" _ "github.com/cs3org/reva/pkg/storage/fs/s3" // Add your own here diff --git a/pkg/storage/fs/ocis/grants.go b/pkg/storage/fs/ocis/grants.go new file mode 100644 index 00000000000..e08e52045ea --- /dev/null +++ b/pkg/storage/fs/ocis/grants.go @@ -0,0 +1,134 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "path/filepath" + "strings" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/storage/utils/ace" + "github.com/pkg/xattr" +) + +func (fs *ocisfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + log := appctx.GetLogger(ctx) + log.Debug().Interface("ref", ref).Interface("grant", g).Msg("AddGrant()") + var node *Node + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + + np := filepath.Join(fs.pw.Root, "nodes", node.ID) + e := ace.FromGrant(g) + principal, value := e.Marshal() + if err := xattr.Set(np, sharePrefix+principal, value); err != nil { + return err + } + return fs.tp.Propagate(ctx, node) +} + +func (fs *ocisfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { + var node *Node + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + log := appctx.GetLogger(ctx) + np := filepath.Join(fs.pw.Root, "nodes", node.ID) + var attrs []string + if attrs, err = xattr.List(np); err != nil { + log.Error().Err(err).Msg("error listing attributes") + return nil, err + } + + log.Debug().Interface("attrs", attrs).Msg("read attributes") + + aces := extractACEsFromAttrs(ctx, np, attrs) + + grants = make([]*provider.Grant, 0, len(aces)) + for i := range aces { + grants = append(grants, aces[i].Grant()) + } + + return grants, nil +} + +func (fs *ocisfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { + var node *Node + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + + var attr string + if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { + attr = sharePrefix + "g:" + g.Grantee.Id.OpaqueId + } else { + attr = sharePrefix + "u:" + g.Grantee.Id.OpaqueId + } + + np := filepath.Join(fs.pw.Root, "nodes", node.ID) + if err = xattr.Remove(np, attr); err != nil { + return + } + + return fs.tp.Propagate(ctx, node) +} + +func (fs *ocisfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { + return fs.AddGrant(ctx, ref, g) +} + +// extractACEsFromAttrs reads ACEs in the list of attrs from the node +func extractACEsFromAttrs(ctx context.Context, fsfn string, attrs []string) (entries []*ace.ACE) { + log := appctx.GetLogger(ctx) + entries = []*ace.ACE{} + for i := range attrs { + if strings.HasPrefix(attrs[i], sharePrefix) { + var value []byte + var err error + if value, err = xattr.Get(fsfn, attrs[i]); err != nil { + log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute") + continue + } + var e *ace.ACE + principal := attrs[i][len(sharePrefix):] + if e, err = ace.Unmarshal(principal, value); err != nil { + log.Error().Err(err).Str("principal", principal).Str("attr", attrs[i]).Msg("could unmarshal ace") + continue + } + entries = append(entries, e) + } + } + return +} diff --git a/pkg/storage/fs/ocis/metadata.go b/pkg/storage/fs/ocis/metadata.go new file mode 100644 index 00000000000..e8f94beb52a --- /dev/null +++ b/pkg/storage/fs/ocis/metadata.go @@ -0,0 +1,69 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "path/filepath" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/pkg/errors" + "github.com/pkg/xattr" +) + +func (fs *ocisfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { + n, err := fs.pw.NodeFromResource(ctx, ref) + if err != nil { + return errors.Wrap(err, "ocisfs: error resolving ref") + } + + if !n.Exists { + err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) + return err + } + nodePath := filepath.Join(fs.pw.Root, "nodes", n.ID) + for k, v := range md.Metadata { + attrName := metadataPrefix + k + if err = xattr.Set(nodePath, attrName, []byte(v)); err != nil { + return errors.Wrap(err, "ocisfs: could not set metadata attribute "+attrName+" to "+k) + } + } + return +} + +func (fs *ocisfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { + n, err := fs.pw.NodeFromResource(ctx, ref) + if err != nil { + return errors.Wrap(err, "ocisfs: error resolving ref") + } + + if !n.Exists { + err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) + return err + } + nodePath := filepath.Join(fs.pw.Root, "nodes", n.ID) + for i := range keys { + attrName := metadataPrefix + keys[i] + if err = xattr.Remove(nodePath, attrName); err != nil { + return errors.Wrap(err, "ocisfs: could not remove metadata attribute "+attrName) + } + } + return +} diff --git a/pkg/storage/fs/ocis/node.go b/pkg/storage/fs/ocis/node.go new file mode 100644 index 00000000000..eae9e4484be --- /dev/null +++ b/pkg/storage/fs/ocis/node.go @@ -0,0 +1,292 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/mime" + "github.com/pkg/errors" + "github.com/pkg/xattr" +) + +// Node represents a node in the tree and provides methods to get a Parent or Child instance +type Node struct { + pw *Path + ParentID string + ID string + Name string + ownerID string // used to cache the owner id + ownerIDP string // used to cache the owner idp + Exists bool +} + +func (n *Node) writeMetadata(owner *userpb.UserId) (err error) { + nodePath := filepath.Join(n.pw.Root, "nodes", n.ID) + if err = xattr.Set(nodePath, "user.ocis.parentid", []byte(n.ParentID)); err != nil { + return errors.Wrap(err, "ocisfs: could not set parentid attribute") + } + if err = xattr.Set(nodePath, "user.ocis.name", []byte(n.Name)); err != nil { + return errors.Wrap(err, "ocisfs: could not set name attribute") + } + if owner != nil { + if err = xattr.Set(nodePath, "user.ocis.owner.id", []byte(owner.OpaqueId)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner id attribute") + } + if err = xattr.Set(nodePath, "user.ocis.owner.idp", []byte(owner.Idp)); err != nil { + return errors.Wrap(err, "ocisfs: could not set owner idp attribute") + } + } + return +} + +// ReadNode creates a new instance from an id and checks if it exists +func ReadNode(ctx context.Context, pw *Path, id string) (n *Node, err error) { + n = &Node{ + pw: pw, + ID: id, + } + + nodePath := filepath.Join(n.pw.Root, "nodes", n.ID) + + // lookup parent id in extended attributes + var attrBytes []byte + if attrBytes, err = xattr.Get(nodePath, "user.ocis.parentid"); err == nil { + n.ParentID = string(attrBytes) + } else { + return + } + // lookup name in extended attributes + if attrBytes, err = xattr.Get(nodePath, "user.ocis.name"); err == nil { + n.Name = string(attrBytes) + } else { + return + } + + var root *Node + if root, err = pw.HomeOrRootNode(ctx); err != nil { + return + } + parentID := n.ParentID + + log := appctx.GetLogger(ctx) + for parentID != root.ID { + log.Debug().Interface("node", n).Str("root.ID", root.ID).Msg("ReadNode()") + // walk to root to check node is not part of a deleted subtree + parentPath := filepath.Join(n.pw.Root, "nodes", parentID) + + if attrBytes, err = xattr.Get(parentPath, "user.ocis.parentid"); err == nil { + parentID = string(attrBytes) + log.Debug().Interface("node", n).Str("root.ID", root.ID).Str("parentID", parentID).Msg("ReadNode() found parent") + } else { + log.Error().Err(err).Interface("node", n).Str("root.ID", root.ID).Msg("ReadNode()") + if os.IsNotExist(err) { + return + } + return + } + } + + n.Exists = true + log.Debug().Interface("node", n).Msg("ReadNode() found node") + + return +} + +// Child returns the child node with the given name +func (n *Node) Child(name string) (c *Node, err error) { + c = &Node{ + pw: n.pw, + ParentID: n.ID, + Name: name, + } + var link string + if link, err = os.Readlink(filepath.Join(n.pw.Root, "nodes", n.ID, name)); os.IsNotExist(err) { + err = nil // if the file does not exist we return a node that has Exists = false + return + } + if err != nil { + err = errors.Wrap(err, "ocisfs: Wrap: readlink error") + return + } + if strings.HasPrefix(link, "../") { + c.Exists = true + c.ID = filepath.Base(link) + } else { + err = fmt.Errorf("ocisfs: expected '../ prefix, got' %+v", link) + } + return +} + +// Parent returns the parent node +func (n *Node) Parent() (p *Node, err error) { + if n.ParentID == "" { + return nil, fmt.Errorf("ocisfs: root has no parent") + } + p = &Node{ + pw: n.pw, + ID: n.ParentID, + } + + parentPath := filepath.Join(n.pw.Root, "nodes", n.ParentID) + + // lookup parent id in extended attributes + var attrBytes []byte + if attrBytes, err = xattr.Get(parentPath, "user.ocis.parentid"); err == nil { + p.ParentID = string(attrBytes) + } else { + return + } + // lookup name in extended attributes + if attrBytes, err = xattr.Get(parentPath, "user.ocis.name"); err == nil { + p.Name = string(attrBytes) + } else { + return + } + + // check node exists + if _, err := os.Stat(parentPath); err == nil { + p.Exists = true + } + return +} + +// Owner returns the cached owner id or reads it from the extended attributes +// TODO can be private as only the AsResourceInfo uses it +func (n *Node) Owner() (id string, idp string, err error) { + if n.ownerID != "" && n.ownerIDP != "" { + return n.ownerID, n.ownerIDP, nil + } + + nodePath := filepath.Join(n.pw.Root, "nodes", n.ParentID) + // lookup parent id in extended attributes + var attrBytes []byte + // lookup name in extended attributes + if attrBytes, err = xattr.Get(nodePath, "user.ocis.owner.id"); err == nil { + n.ownerID = string(attrBytes) + } else { + return + } + // lookup name in extended attributes + if attrBytes, err = xattr.Get(nodePath, "user.ocis.owner.idp"); err == nil { + n.ownerIDP = string(attrBytes) + } else { + return + } + return n.ownerID, n.ownerIDP, err +} + +// AsResourceInfo return the node as CS3 ResourceInfo +func (n *Node) AsResourceInfo(ctx context.Context) (ri *provider.ResourceInfo, err error) { + log := appctx.GetLogger(ctx) + + var fn string + nodePath := filepath.Join(n.pw.Root, "nodes", n.ID) + + var fi os.FileInfo + + nodeType := provider.ResourceType_RESOURCE_TYPE_INVALID + if fi, err = os.Lstat(nodePath); err != nil { + return + } + + var target []byte + switch { + case fi.IsDir(): + if target, err = xattr.Get(nodePath, referenceAttr); err == nil { + nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE + } else { + nodeType = provider.ResourceType_RESOURCE_TYPE_CONTAINER + } + case fi.Mode().IsRegular(): + nodeType = provider.ResourceType_RESOURCE_TYPE_FILE + case fi.Mode()&os.ModeSymlink != 0: + nodeType = provider.ResourceType_RESOURCE_TYPE_SYMLINK + // TODO reference using ext attr on a symlink + // nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE + } + + var etag []byte + // TODO optionally store etag in new `root/attributes/` file + if etag, err = xattr.Get(nodePath, "user.ocis.etag"); err != nil { + log.Error().Err(err).Interface("node", n).Msg("could not read etag") + } + + id := &provider.ResourceId{OpaqueId: n.ID} + + fn, err = n.pw.Path(ctx, n) + if err != nil { + return nil, err + } + ri = &provider.ResourceInfo{ + Id: id, + Path: fn, + Type: nodeType, + Etag: string(etag), + MimeType: mime.Detect(nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER, fn), + Size: uint64(fi.Size()), + // TODO fix permissions + PermissionSet: &provider.ResourcePermissions{ListContainer: true, CreateContainer: true}, + Mtime: &types.Timestamp{ + Seconds: uint64(fi.ModTime().Unix()), + // TODO read nanos from where? Nanos: fi.MTimeNanos, + }, + Target: string(target), + } + + if owner, idp, err := n.Owner(); err == nil { + ri.Owner = &userpb.UserId{ + Idp: idp, + OpaqueId: owner, + } + } + + // TODO only read the requested metadata attributes + if attrs, err := xattr.List(nodePath); err == nil { + ri.ArbitraryMetadata = &provider.ArbitraryMetadata{ + Metadata: map[string]string{}, + } + for i := range attrs { + if strings.HasPrefix(attrs[i], metadataPrefix) { + k := strings.TrimPrefix(attrs[i], metadataPrefix) + if v, err := xattr.Get(nodePath, attrs[i]); err == nil { + ri.ArbitraryMetadata.Metadata[k] = string(v) + } else { + log.Error().Err(err).Interface("node", n).Str("attr", attrs[i]).Msg("could not get attribute value") + } + } + } + } else { + log.Error().Err(err).Interface("node", n).Msg("could not list attributes") + } + + log.Debug(). + Interface("ri", ri). + Msg("AsResourceInfo") + + return ri, nil +} diff --git a/pkg/storage/fs/ocis/ocis.go b/pkg/storage/fs/ocis/ocis.go new file mode 100644 index 00000000000..2b614e8766e --- /dev/null +++ b/pkg/storage/fs/ocis/ocis.go @@ -0,0 +1,361 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "io" + "net/url" + "os" + "path/filepath" + "strings" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/storage" + "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/utils/templates" + "github.com/cs3org/reva/pkg/user" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "github.com/pkg/xattr" +) + +const ( + // TODO the below comment is currently copied from the owncloud driver, revisit + // Currently,extended file attributes have four separated + // namespaces (user, trusted, security and system) followed by a dot. + // A non root user can only manipulate the user. namespace, which is what + // we will use to store ownCloud specific metadata. To prevent name + // collisions with other apps We are going to introduce a sub namespace + // "user.ocis." + + // SharePrefix is the prefix for sharing related extended attributes + sharePrefix string = "user.ocis.acl." + metadataPrefix string = "user.ocis.md." + // TODO implement favorites metadata flag + //favPrefix string = "user.ocis.fav." // favorite flag, per user + // TODO use etag prefix instead of single etag property + //etagPrefix string = "user.ocis.etag." // allow overriding a calculated etag with one from the extended attributes + referenceAttr string = "user.ocis.cs3.ref" // arbitrary metadata + //checksumPrefix string = "user.ocis.cs." // TODO add checksum support + trashOriginAttr string = "user.ocis.trash.origin" // trash origin +) + +func init() { + registry.Register("ocis", New) +} + +func parseConfig(m map[string]interface{}) (*Path, error) { + pw := &Path{} + if err := mapstructure.Decode(m, pw); err != nil { + err = errors.Wrap(err, "error decoding conf") + return nil, err + } + return pw, nil +} + +func (pw *Path) init(m map[string]interface{}) { + if pw.UserLayout == "" { + pw.UserLayout = "{{.Id.OpaqueId}}" + } + // ensure user layout has no starting or trailing / + pw.UserLayout = strings.Trim(pw.UserLayout, "/") + + if pw.ShareFolder == "" { + pw.ShareFolder = "/Shares" + } + // ensure share folder always starts with slash + pw.ShareFolder = filepath.Join("/", pw.ShareFolder) + + // c.DataDirectory should never end in / unless it is the root + pw.Root = filepath.Clean(pw.Root) +} + +// New returns an implementation to of the storage.FS interface that talk to +// a local filesystem. +func New(m map[string]interface{}) (storage.FS, error) { + pw, err := parseConfig(m) + if err != nil { + return nil, err + } + pw.init(m) + + dataPaths := []string{ + filepath.Join(pw.Root, "nodes"), + // notes contain symlinks from nodes//uploads/ to ../../uploads/ + // better to keep uploads on a fast / volatile storage before a workflow finally moves them to the nodes dir + filepath.Join(pw.Root, "uploads"), + filepath.Join(pw.Root, "trash"), + } + for _, v := range dataPaths { + if err := os.MkdirAll(v, 0700); err != nil { + logger.New().Error().Err(err). + Str("path", v). + Msg("could not create data dir") + } + } + + // the root node has an empty name, or use `.` ? + // the root node has no parent, or use `root` ? + if err = createNode(&Node{pw: pw, ID: "root"}, nil); err != nil { + return nil, err + } + + tp, err := NewTree(pw) + if err != nil { + return nil, err + } + + return &ocisfs{ + tp: tp, + pw: pw, + }, nil +} + +type ocisfs struct { + tp TreePersistence + pw *Path +} + +func (fs *ocisfs) Shutdown(ctx context.Context) error { + return nil +} + +func (fs *ocisfs) GetQuota(ctx context.Context) (int, int, error) { + return 0, 0, nil +} + +// CreateHome creates a new root node that has no parent id +func (fs *ocisfs) CreateHome(ctx context.Context) (err error) { + if !fs.pw.EnableHome || fs.pw.UserLayout == "" { + return errtypes.NotSupported("ocisfs: CreateHome() home supported disabled") + } + + var n *Node + if n, err = fs.pw.RootNode(ctx); err != nil { + return + } + _, err = fs.pw.WalkPath(ctx, n, fs.pw.mustGetUserLayout(ctx), func(ctx context.Context, n *Node) error { + if !n.Exists { + if err := fs.tp.CreateDir(ctx, n); err != nil { + return err + } + } + return nil + }) + return +} + +// GetHome is called to look up the home path for a user +// It is NOT supposed to return the internal path but the external path +func (fs *ocisfs) GetHome(ctx context.Context) (string, error) { + if !fs.pw.EnableHome || fs.pw.UserLayout == "" { + return "", errtypes.NotSupported("ocisfs: GetHome() home supported disabled") + } + u := user.ContextMustGetUser(ctx) + layout := templates.WithUser(u, fs.pw.UserLayout) + return filepath.Join(fs.pw.Root, layout), nil // TODO use a namespace? +} + +// Tree persistence + +// GetPathByID returns the fn pointed by the file id, without the internal namespace +func (fs *ocisfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { + return fs.tp.GetPathByID(ctx, id) +} + +func (fs *ocisfs) CreateDir(ctx context.Context, fn string) (err error) { + var node *Node + if node, err = fs.pw.NodeFromPath(ctx, fn); err != nil { + return + } + if node.Exists { + return errtypes.AlreadyExists(fn) + } + return fs.tp.CreateDir(ctx, node) +} + +// CreateReference creates a reference as a node folder with the target stored in extended attributes +// There is no difference between the /Shares folder and normal nodes because the storage is not supposed to be accessible without the storage provider. +// In effect everything is a shadow namespace. +// To mimic the eos end owncloud driver we only allow references as children of the "/Shares" folder +// TODO when home support is enabled should the "/Shares" folder still be listed? +func (fs *ocisfs) CreateReference(ctx context.Context, p string, targetURI *url.URL) (err error) { + + p = strings.Trim(p, "/") + parts := strings.Split(p, "/") + + if len(parts) != 2 { + return errtypes.PermissionDenied("ocisfs: references must be a child of the share folder: share_folder=" + fs.pw.ShareFolder + " path=" + p) + } + + if parts[0] != strings.Trim(fs.pw.ShareFolder, "/") { + return errtypes.PermissionDenied("ocisfs: cannot create references outside the share folder: share_folder=" + fs.pw.ShareFolder + " path=" + p) + } + + // create Shares folder if it does not exist + var n *Node + if n, err = fs.pw.NodeFromPath(ctx, fs.pw.ShareFolder); err != nil { + return errtypes.InternalError(err.Error()) + } else if !n.Exists { + if err = fs.tp.CreateDir(ctx, n); err != nil { + return + } + } + + if n, err = n.Child(parts[1]); err != nil { + return errtypes.InternalError(err.Error()) + } + + if n.Exists { + // TODO append increasing number to mountpoint name + return errtypes.AlreadyExists(p) + } + + if err = fs.tp.CreateDir(ctx, n); err != nil { + return + } + + internal := filepath.Join(fs.pw.Root, "nodes", n.ID) + if err = xattr.Set(internal, referenceAttr, []byte(targetURI.String())); err != nil { + return errors.Wrapf(err, "ocisfs: error setting the target %s on the reference file %s", targetURI.String(), internal) + } + return nil +} + +func (fs *ocisfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (err error) { + var oldNode, newNode *Node + if oldNode, err = fs.pw.NodeFromResource(ctx, oldRef); err != nil { + return + } + if !oldNode.Exists { + err = errtypes.NotFound(filepath.Join(oldNode.ParentID, oldNode.Name)) + return + } + + if newNode, err = fs.pw.NodeFromResource(ctx, newRef); err != nil { + return + } + return fs.tp.Move(ctx, oldNode, newNode) +} + +func (fs *ocisfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string) (ri *provider.ResourceInfo, err error) { + var node *Node + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + return node.AsResourceInfo(ctx) +} + +func (fs *ocisfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string) (finfos []*provider.ResourceInfo, err error) { + var node *Node + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + var children []*Node + children, err = fs.tp.ListFolder(ctx, node) + if err != nil { + return + } + + for i := range children { + if ri, err := children[i].AsResourceInfo(ctx); err == nil { + finfos = append(finfos, ri) + } + } + return +} + +func (fs *ocisfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { + var node *Node + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + return fs.tp.Delete(ctx, node) +} + +// Data persistence + +func (fs *ocisfs) ContentPath(node *Node) string { + return filepath.Join(fs.pw.Root, "nodes", node.ID) +} + +func (fs *ocisfs) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) { + node, err := fs.pw.NodeFromResource(ctx, ref) + if err != nil { + return nil, errors.Wrap(err, "ocisfs: error resolving ref") + } + + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return nil, err + } + + contentPath := fs.ContentPath(node) + + r, err := os.Open(contentPath) + if err != nil { + if os.IsNotExist(err) { + return nil, errtypes.NotFound(contentPath) + } + return nil, errors.Wrap(err, "ocisfs: error reading "+contentPath) + } + return r, nil +} + +// arbitrary metadata persistence in metadata.go + +// Version persistence in revisions.go + +// Trash persistence in recycle.go + +// share persistence in grants.go + +func (fs *ocisfs) copyMD(s string, t string) (err error) { + var attrs []string + if attrs, err = xattr.List(s); err != nil { + return err + } + for i := range attrs { + if strings.HasPrefix(attrs[i], "user.ocis.") { + var d []byte + if d, err = xattr.Get(s, attrs[i]); err != nil { + return err + } + if err = xattr.Set(t, attrs[i], d); err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/storage/fs/ocis/path.go b/pkg/storage/fs/ocis/path.go new file mode 100644 index 00000000000..2a839dd51e9 --- /dev/null +++ b/pkg/storage/fs/ocis/path.go @@ -0,0 +1,168 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/storage/utils/templates" + "github.com/cs3org/reva/pkg/user" +) + +// Path implements transformations from filepath to node and back +type Path struct { + // ocis fs works on top of a dir of uuid nodes + Root string `mapstructure:"root"` + + // UserLayout describes the relative path from the storage's root node to the users home node. + UserLayout string `mapstructure:"user_layout"` + + // TODO NodeLayout option to save nodes as eg. nodes/1d/d8/1dd84abf-9466-4e14-bb86-02fc4ea3abcf + + // EnableHome enables the creation of home directories. + EnableHome bool `mapstructure:"enable_home"` + ShareFolder string `mapstructure:"share_folder"` +} + +// NodeFromResource takes in a request path or request id and converts it to a Node +func (pw *Path) NodeFromResource(ctx context.Context, ref *provider.Reference) (*Node, error) { + if ref.GetPath() != "" { + return pw.NodeFromPath(ctx, ref.GetPath()) + } + + if ref.GetId() != nil { + return pw.NodeFromID(ctx, ref.GetId()) + } + + // reference is invalid + return nil, fmt.Errorf("invalid reference %+v", ref) +} + +// NodeFromPath converts a filename into a Node +func (pw *Path) NodeFromPath(ctx context.Context, fn string) (node *Node, err error) { + log := appctx.GetLogger(ctx) + log.Debug().Interface("fn", fn).Msg("NodeFromPath()") + + if node, err = pw.HomeOrRootNode(ctx); err != nil { + return + } + + if fn != "/" { + node, err = pw.WalkPath(ctx, node, fn, func(ctx context.Context, n *Node) error { + log.Debug().Interface("node", n).Msg("NodeFromPath() walk") + return nil + }) + } + + return +} + +// NodeFromID returns the internal path for the id +func (pw *Path) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *Node, err error) { + if id == nil || id.OpaqueId == "" { + return nil, fmt.Errorf("invalid resource id %+v", id) + } + return ReadNode(ctx, pw, id.OpaqueId) +} + +// Path returns the path for node +func (pw *Path) Path(ctx context.Context, n *Node) (p string, err error) { + var root *Node + if root, err = pw.HomeOrRootNode(ctx); err != nil { + return + } + for n.ID != root.ID { + p = filepath.Join(n.Name, p) + if n, err = n.Parent(); err != nil { + appctx.GetLogger(ctx). + Error().Err(err). + Str("path", p). + Interface("node", n). + Msg("Path()") + return + } + } + return +} + +// RootNode returns the root node of the storage +func (pw *Path) RootNode(ctx context.Context) (node *Node, err error) { + return &Node{ + pw: pw, + ID: "root", + Name: "", + ParentID: "", + Exists: true, + }, nil +} + +// HomeNode returns the home node of a user +func (pw *Path) HomeNode(ctx context.Context) (node *Node, err error) { + if !pw.EnableHome { + return nil, errtypes.NotSupported("ocisfs: home supported disabled") + } + + if node, err = pw.RootNode(ctx); err != nil { + return + } + node, err = pw.WalkPath(ctx, node, pw.mustGetUserLayout(ctx), nil) + return +} + +// WalkPath calls n.Child(segment) on every path segment in p starting at the node r +// If a function f is given it will be executed for every segment node, but not the root node r +func (pw *Path) WalkPath(ctx context.Context, r *Node, p string, f func(ctx context.Context, n *Node) error) (*Node, error) { + segments := strings.Split(strings.Trim(p, "/"), "/") + var err error + for i := range segments { + if r, err = r.Child(segments[i]); err != nil { + return r, err + } + // if an intermediate node is missing return not found + if !r.Exists && i < len(segments)-1 { + return r, errtypes.NotFound(segments[i]) + } + if f != nil { + if err = f(ctx, r); err != nil { + return r, err + } + } + } + return r, nil +} + +// HomeOrRootNode returns the users home node when home support is enabled. +// it returns the storages root node otherwise +func (pw *Path) HomeOrRootNode(ctx context.Context) (node *Node, err error) { + if pw.EnableHome { + return pw.HomeNode(ctx) + } + return pw.RootNode(ctx) +} + +func (pw *Path) mustGetUserLayout(ctx context.Context) string { + u := user.ContextMustGetUser(ctx) + return templates.WithUser(u, pw.UserLayout) +} diff --git a/pkg/storage/fs/ocis/persistence.go b/pkg/storage/fs/ocis/persistence.go new file mode 100644 index 00000000000..5b9dcc16fc7 --- /dev/null +++ b/pkg/storage/fs/ocis/persistence.go @@ -0,0 +1,59 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "os" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +// TreePersistence is used to manage a tree hierarchy +type TreePersistence interface { + GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) + GetMD(ctx context.Context, node *Node) (os.FileInfo, error) + ListFolder(ctx context.Context, node *Node) ([]*Node, error) + //CreateHome(owner *userpb.UserId) (n *Node, err error) + CreateDir(ctx context.Context, node *Node) (err error) + //CreateReference(ctx context.Context, node *Node, targetURI *url.URL) error + Move(ctx context.Context, oldNode *Node, newNode *Node) (err error) + Delete(ctx context.Context, node *Node) (err error) + + Propagate(ctx context.Context, node *Node) (err error) +} + +// PathWrapper is used to encapsulate path transformations +type PathWrapper interface { + NodeFromResource(ctx context.Context, ref *provider.Reference) (node *Node, err error) + NodeFromID(ctx context.Context, id *provider.ResourceId) (node *Node, err error) + NodeFromPath(ctx context.Context, fn string) (node *Node, err error) + Path(ctx context.Context, node *Node) (path string, err error) + + // HomeNode returns the currently logged in users home node + // requires EnableHome to be true + HomeNode(ctx context.Context) (node *Node, err error) + + // RootNode returns the storage root node + RootNode(ctx context.Context) (node *Node, err error) + + // HomeOrRootNode returns the users home node when home support is enabled. + // it returns the storages root node otherwise + HomeOrRootNode(ctx context.Context) (node *Node, err error) +} diff --git a/pkg/storage/fs/ocis/recycle.go b/pkg/storage/fs/ocis/recycle.go new file mode 100644 index 00000000000..359561a4085 --- /dev/null +++ b/pkg/storage/fs/ocis/recycle.go @@ -0,0 +1,233 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "os" + "path/filepath" + "strings" + "time" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/user" + "github.com/pkg/errors" + "github.com/pkg/xattr" +) + +// Recycle items are stored inside the node folder and start with the uuid of the deleted node. +// The `.T.` indicates it is a trash item and what follows is the timestamp of the deletion. +// The deleted file is kept in the same location/dir as the original node. This prevents deletes +// from triggering cross storage moves when the trash is accidentally stored on another partition, +// because the admin mounted a different partition there. +// TODO For an efficient listing of deleted nodes the ocis storages trash folder should have +// contain a directory with symlinks to trash files for every userid/"root" + +func (fs *ocisfs) ListRecycle(ctx context.Context) (items []*provider.RecycleItem, err error) { + log := appctx.GetLogger(ctx) + + trashRoot := fs.getRecycleRoot(ctx) + + items = make([]*provider.RecycleItem, 0) + + f, err := os.Open(trashRoot) + if err != nil { + if os.IsNotExist(err) { + return items, nil + } + return nil, errors.Wrap(err, "tree: error listing "+trashRoot) + } + + names, err := f.Readdirnames(0) + if err != nil { + return nil, err + } + for i := range names { + var link string + link, err = os.Readlink(filepath.Join(trashRoot, names[i])) + if err != nil { + log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Msg("error reading trash link, skipping") + err = nil + continue + } + parts := strings.SplitN(filepath.Base(link), ".T.", 2) + if len(parts) != 2 { + log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("link", link).Interface("parts", parts).Msg("malformed trash link, skipping") + continue + } + + nodePath := filepath.Join(fs.pw.Root, "nodes", filepath.Base(link)) + md, err := os.Stat(nodePath) + if err != nil { + log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("link", link).Interface("parts", parts).Msg("could not stat trash item, skipping") + continue + } + item := &provider.RecycleItem{ + Type: getResourceType(md.IsDir()), + Size: uint64(md.Size()), + Key: filepath.Base(trashRoot) + ":" + parts[0], // glue using :, a / is interpreted as a path and only the node id will reach the other methods + } + if deletionTime, err := time.Parse(time.RFC3339Nano, parts[1]); err == nil { + item.DeletionTime = &types.Timestamp{ + Seconds: uint64(deletionTime.Unix()), + // TODO nanos + } + } else { + log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("link", link).Interface("parts", parts).Msg("could parse time format, ignoring") + } + + // lookup parent id in extended attributes + var attrBytes []byte + if attrBytes, err = xattr.Get(nodePath, trashOriginAttr); err == nil { + item.Path = string(attrBytes) + } else { + log.Error().Err(err).Str("trashRoot", trashRoot).Str("name", names[i]).Str("link", link).Msg("could not read origin path, skipping") + continue + } + + items = append(items, item) + } + return +} + +func (fs *ocisfs) RestoreRecycleItem(ctx context.Context, key string) (err error) { + log := appctx.GetLogger(ctx) + + if key == "" { + return errtypes.InternalError("key is empty") + } + + kp := strings.SplitN(key, ":", 2) + if len(kp) != 2 { + log.Error().Err(err).Str("key", key).Msg("malformed key") + return + } + trashItem := filepath.Join(fs.pw.Root, "trash", kp[0], kp[1]) + + var link string + link, err = os.Readlink(trashItem) + if err != nil { + log.Error().Err(err).Str("trashItem", trashItem).Msg("error reading trash link") + return + } + parts := strings.SplitN(filepath.Base(link), ".T.", 2) + if len(parts) != 2 { + log.Error().Err(err).Str("trashItem", trashItem).Interface("parts", parts).Msg("malformed trash link") + return + } + + deletedNodePath := filepath.Join(fs.pw.Root, "nodes", filepath.Base(link)) + + // get origin node + origin := "/" + + // lookup parent id in extended attributes + var attrBytes []byte + if attrBytes, err = xattr.Get(deletedNodePath, trashOriginAttr); err == nil { + origin = string(attrBytes) + } else { + log.Error().Err(err).Str("trashItem", trashItem).Str("link", link).Str("deletedNodePath", deletedNodePath).Msg("could not read origin path, restoring to /") + } + + // link to origin + var n *Node + n, err = fs.pw.NodeFromPath(ctx, origin) + if err != nil { + return + } + + if n.Exists { + return errtypes.AlreadyExists("origin already exists") + } + + // rename to node only name, so it is picked up by id + nodePath := filepath.Join(fs.pw.Root, "nodes", parts[0]) + err = os.Rename(deletedNodePath, nodePath) + if err != nil { + return + } + + // add the entry for the parent dir + err = os.Symlink("../"+parts[0], filepath.Join(fs.pw.Root, "nodes", n.ParentID, n.Name)) + if err != nil { + return + } + n.Exists = true + + // delete item link in trash + if err = os.Remove(trashItem); err != nil { + log.Error().Err(err).Str("trashItem", trashItem).Msg("error deleting trashitem") + } + return fs.tp.Propagate(ctx, n) + +} + +func (fs *ocisfs) PurgeRecycleItem(ctx context.Context, key string) (err error) { + log := appctx.GetLogger(ctx) + + kp := strings.SplitN(key, ":", 2) + if len(kp) != 2 { + log.Error().Str("key", key).Msg("malformed key") + return + } + trashItem := filepath.Join(fs.pw.Root, "trash", kp[0], kp[1]) + + var link string + link, err = os.Readlink(trashItem) + if err != nil { + log.Error().Err(err).Str("trashItem", trashItem).Msg("error reading trash link") + return + } + + // delete trash node link in nodes dir + deletedNodePath := filepath.Join(fs.pw.Root, "nodes", filepath.Base(link)) + if err = os.Remove(deletedNodePath); err != nil { + log.Error().Err(err).Str("deletedNodePath", deletedNodePath).Msg("error deleting trash node") + return + } + + // delete item link in trash + if err = os.Remove(trashItem); err != nil { + log.Error().Err(err).Str("trashItem", trashItem).Msg("error deleting trash item") + } + return +} + +func (fs *ocisfs) EmptyRecycle(ctx context.Context) error { + return os.RemoveAll(fs.getRecycleRoot(ctx)) +} + +func getResourceType(isDir bool) provider.ResourceType { + if isDir { + return provider.ResourceType_RESOURCE_TYPE_CONTAINER + } + return provider.ResourceType_RESOURCE_TYPE_FILE +} + +func (fs *ocisfs) getRecycleRoot(ctx context.Context) string { + if fs.pw.EnableHome { + u := user.ContextMustGetUser(ctx) + // TODO use layout, see Tree.Delete() for problem + return filepath.Join(fs.pw.Root, "trash", u.Id.OpaqueId) + } + return filepath.Join(fs.pw.Root, "trash", "root") +} diff --git a/pkg/storage/fs/ocis/revisions.go b/pkg/storage/fs/ocis/revisions.go new file mode 100644 index 00000000000..50b8164e830 --- /dev/null +++ b/pkg/storage/fs/ocis/revisions.go @@ -0,0 +1,149 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "io" + "os" + "path/filepath" + "strings" + "time" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/pkg/errors" +) + +// Revision entries are stored inside the node folder and start with the same uuid as the current version. +// The `.REV.` indicates it is a revision and what follows is a timestamp, so multiple versions +// can be kept in the same location as the current file content. This prevents new fileuploads +// to trigger cross storage moves when revisions accidentally are stored on another partition, +// because the admin mounted a different partition there. +// We can add a background process to move old revisions to a slower storage +// and replace the revision file with a symbolic link in the future, if necessary. + +func (fs *ocisfs) ListRevisions(ctx context.Context, ref *provider.Reference) (revisions []*provider.FileVersion, err error) { + var node *Node + if node, err = fs.pw.NodeFromResource(ctx, ref); err != nil { + return + } + if !node.Exists { + err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) + return + } + + revisions = []*provider.FileVersion{} + nodePath := filepath.Join(fs.pw.Root, "nodes", node.ID) + if items, err := filepath.Glob(nodePath + ".REV.*"); err == nil { + for i := range items { + if fi, err := os.Stat(items[i]); err == nil { + rev := &provider.FileVersion{ + Key: filepath.Base(items[i]), + Size: uint64(fi.Size()), + Mtime: uint64(fi.ModTime().Unix()), + } + revisions = append(revisions, rev) + } + } + } + return +} + +func (fs *ocisfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (io.ReadCloser, error) { + log := appctx.GetLogger(ctx) + + // verify revision key format + kp := strings.SplitN(revisionKey, ".REV.", 2) + if len(kp) != 2 { + log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey") + return nil, errtypes.NotFound(revisionKey) + } + log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision") + + // check if the node is available and has not been deleted + nodePath := filepath.Join(fs.pw.Root, "nodes", kp[0]) + if _, err := os.Stat(nodePath); err != nil { + if os.IsNotExist(err) { + return nil, errtypes.NotFound(nodePath) + } + return nil, errors.Wrap(err, "ocisfs: error stating node "+kp[0]) + } + + contentPath := filepath.Join(fs.pw.Root, "nodes", revisionKey) + + r, err := os.Open(contentPath) + if err != nil { + if os.IsNotExist(err) { + return nil, errtypes.NotFound(contentPath) + } + return nil, errors.Wrap(err, "ocisfs: error opening revision "+revisionKey) + } + return r, nil +} + +func (fs *ocisfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (err error) { + log := appctx.GetLogger(ctx) + + // verify revision key format + kp := strings.SplitN(revisionKey, ".REV.", 2) + if len(kp) != 2 { + log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey") + return errtypes.NotFound(revisionKey) + } + + // move current version to new revision + nodePath := filepath.Join(fs.pw.Root, "nodes", kp[0]) + var fi os.FileInfo + if fi, err = os.Stat(nodePath); err == nil { + // versions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries + versionsPath := filepath.Join(fs.pw.Root, "nodes", kp[0]+".REV."+fi.ModTime().UTC().Format(time.RFC3339Nano)) + + err = os.Rename(nodePath, versionsPath) + if err != nil { + return + } + + // copy old revision to current location + + revisionPath := filepath.Join(fs.pw.Root, "nodes", revisionKey) + var revision, destination *os.File + revision, err = os.Open(revisionPath) + if err != nil { + return + } + defer revision.Close() + + destination, err = os.OpenFile(nodePath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return + } + defer destination.Close() + _, err = io.Copy(destination, revision) + if err != nil { + return + } + + return fs.copyMD(revisionPath, nodePath) + } + + log.Error().Err(err).Interface("ref", ref).Str("revisionKey", revisionKey).Msg("original node does not exist") + return +} diff --git a/pkg/storage/fs/ocis/tree.go b/pkg/storage/fs/ocis/tree.go new file mode 100644 index 00000000000..553df7b3309 --- /dev/null +++ b/pkg/storage/fs/ocis/tree.go @@ -0,0 +1,317 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "encoding/hex" + "math/rand" + "os" + "path/filepath" + "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/user" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/pkg/xattr" +) + +// Tree manages a hierarchical tree +type Tree struct { + pw *Path +} + +// NewTree creates a new Tree instance +func NewTree(pw *Path) (TreePersistence, error) { + return &Tree{ + pw: pw, + }, nil +} + +// GetMD returns the metadata of a node in the tree +func (t *Tree) GetMD(ctx context.Context, node *Node) (os.FileInfo, error) { + md, err := os.Stat(filepath.Join(t.pw.Root, "nodes", node.ID)) + if err != nil { + if os.IsNotExist(err) { + return nil, errtypes.NotFound(node.ID) + } + return nil, errors.Wrap(err, "tree: error stating "+node.ID) + } + + return md, nil +} + +// GetPathByID returns the fn pointed by the file id, without the internal namespace +func (t *Tree) GetPathByID(ctx context.Context, id *provider.ResourceId) (relativeExternalPath string, err error) { + var node *Node + node, err = t.pw.NodeFromID(ctx, id) + if err != nil { + return + } + + relativeExternalPath, err = t.pw.Path(ctx, node) + return +} + +// does not take care of linking back to parent +// TODO check if node exists? +func createNode(n *Node, owner *userpb.UserId) (err error) { + // create a directory node + nodePath := filepath.Join(n.pw.Root, "nodes", n.ID) + if err = os.MkdirAll(nodePath, 0700); err != nil { + return errors.Wrap(err, "ocisfs: error creating node") + } + + return n.writeMetadata(owner) +} + +// CreateDir creates a new directory entry in the tree +func (t *Tree) CreateDir(ctx context.Context, node *Node) (err error) { + + if node.Exists || node.ID != "" { + return errtypes.AlreadyExists(node.ID) // path? + } + + // create a directory node + node.ID = uuid.New().String() + + if t.pw.EnableHome { + if u, ok := user.ContextGetUser(ctx); ok { + err = createNode(node, u.Id) + } else { + log := appctx.GetLogger(ctx) + log.Error().Msg("home support enabled but no user in context") + err = errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") + } + } else { + err = createNode(node, nil) + } + if err != nil { + return nil + } + + // make child appear in listings + err = os.Symlink("../"+node.ID, filepath.Join(t.pw.Root, "nodes", node.ParentID, node.Name)) + if err != nil { + return + } + return t.Propagate(ctx, node) +} + +// Move replaces the target with the source +func (t *Tree) Move(ctx context.Context, oldNode *Node, newNode *Node) (err error) { + // if target exists delete it without trashing it + if newNode.Exists { + // TODO make sure all children are deleted + if err := os.RemoveAll(filepath.Join(t.pw.Root, "nodes", newNode.ID)); err != nil { + return errors.Wrap(err, "ocisfs: Move: error deleting target node "+newNode.ID) + } + } + // are we just renaming (parent stays the same)? + if oldNode.ParentID == newNode.ParentID { + + parentPath := filepath.Join(t.pw.Root, "nodes", oldNode.ParentID) + + // rename child + err = os.Rename( + filepath.Join(parentPath, oldNode.Name), + filepath.Join(parentPath, newNode.Name), + ) + if err != nil { + return errors.Wrap(err, "ocisfs: could not rename child") + } + + tgtPath := filepath.Join(t.pw.Root, "nodes", newNode.ID) + + // update name attribute + if err := xattr.Set(tgtPath, "user.ocis.name", []byte(newNode.Name)); err != nil { + return errors.Wrap(err, "ocisfs: could not set name attribute") + } + + return t.Propagate(ctx, newNode) + } + + // we are moving the node to a new parent, any target has been removed + // bring old node to the new parent + + // rename child + err = os.Rename( + filepath.Join(t.pw.Root, "nodes", oldNode.ParentID, oldNode.Name), + filepath.Join(t.pw.Root, "nodes", newNode.ParentID, newNode.Name), + ) + if err != nil { + return errors.Wrap(err, "ocisfs: could not move child") + } + + // update parentid and name + tgtPath := filepath.Join(t.pw.Root, "nodes", newNode.ID) + + if err := xattr.Set(tgtPath, "user.ocis.parentid", []byte(newNode.ParentID)); err != nil { + return errors.Wrap(err, "ocisfs: could not set parentid attribute") + } + if err := xattr.Set(tgtPath, "user.ocis.name", []byte(newNode.Name)); err != nil { + return errors.Wrap(err, "ocisfs: could not set name attribute") + } + + // TODO inefficient because we might update several nodes twice, only propagate unchanged nodes? + // collect in a list, then only stat each node once + // also do this in a go routine ... webdav should check the etag async + + err = t.Propagate(ctx, oldNode) + if err != nil { + return errors.Wrap(err, "ocisfs: Move: could not propagate old node") + } + err = t.Propagate(ctx, newNode) + if err != nil { + return errors.Wrap(err, "ocisfs: Move: could not propagate new node") + } + return nil +} + +// ListFolder lists the content of a folder node +func (t *Tree) ListFolder(ctx context.Context, node *Node) ([]*Node, error) { + + dir := filepath.Join(t.pw.Root, "nodes", node.ID) + f, err := os.Open(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, errtypes.NotFound(dir) + } + return nil, errors.Wrap(err, "tree: error listing "+dir) + } + + names, err := f.Readdirnames(0) + if err != nil { + return nil, err + } + nodes := []*Node{} + for i := range names { + link, err := os.Readlink(filepath.Join(dir, names[i])) + if err != nil { + // TODO log + continue + } + n := &Node{ + pw: t.pw, + ParentID: node.ID, + ID: filepath.Base(link), + Name: names[i], + Exists: true, // TODO + } + + nodes = append(nodes, n) + } + return nodes, nil +} + +// Delete deletes a node in the tree +func (t *Tree) Delete(ctx context.Context, node *Node) (err error) { + + // Prepare the trash + // TODO use layout?, but it requires resolving the owners user if the username is used instead of the id. + // the node knows the owner id so we use that for now + ownerid, _, err := node.Owner() + if err != nil { + return + } + if ownerid == "" { + // fall back to root trash + ownerid = "root" + } + err = os.MkdirAll(filepath.Join(t.pw.Root, "trash", ownerid), 0700) + if err != nil { + return + } + + // get the original path + origin, err := t.pw.Path(ctx, node) + if err != nil { + return + } + + // remove the entry from the parent dir + + src := filepath.Join(t.pw.Root, "nodes", node.ParentID, node.Name) + err = os.Remove(src) + if err != nil { + return + } + + // rename the trashed node so it is not picked up when traversing up the tree + nodePath := filepath.Join(t.pw.Root, "nodes", node.ID) + deletionTime := time.Now().UTC().Format(time.RFC3339Nano) + trashPath := nodePath + ".T." + deletionTime + err = os.Rename(nodePath, trashPath) + if err != nil { + return + } + // set origin location in metadata + if err := xattr.Set(trashPath, trashOriginAttr, []byte(origin)); err != nil { + return err + } + + // make node appear in the owners (or root) trash + // parent id and name are stored as extended attributes in the node itself + trashLink := filepath.Join(t.pw.Root, "trash", ownerid, node.ID) + err = os.Symlink("../nodes/"+node.ID+".T."+deletionTime, trashLink) + if err != nil { + return + } + p, err := node.Parent() + if err != nil { + return + } + return t.Propagate(ctx, p) +} + +// Propagate propagates changes to the root of the tree +func (t *Tree) Propagate(ctx context.Context, node *Node) (err error) { + // generate an etag + bytes := make([]byte, 16) + if _, err := rand.Read(bytes); err != nil { + return err + } + // store in extended attribute + etag := hex.EncodeToString(bytes) + var root *Node + + if root, err = t.pw.HomeOrRootNode(ctx); err != nil { + return + } + for err == nil && node.ID != root.ID { // TODO propagate up to where? + if err := xattr.Set(filepath.Join(t.pw.Root, "nodes", node.ID), "user.ocis.etag", []byte(etag)); err != nil { + log := appctx.GetLogger(ctx) + log.Error().Err(err).Msg("error storing file id") + } + // TODO propagate mtime + // TODO size accounting + + if err != nil { + err = errors.Wrap(err, "ocisfs: Propagate: readlink error") + return + } + + node, err = node.Parent() + } + return +} diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go new file mode 100644 index 00000000000..d3678c11c10 --- /dev/null +++ b/pkg/storage/fs/ocis/upload.go @@ -0,0 +1,498 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ocis + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/errtypes" + "github.com/cs3org/reva/pkg/logger" + "github.com/cs3org/reva/pkg/user" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + tusd "github.com/tus/tusd/pkg/handler" +) + +var defaultFilePerm = os.FileMode(0664) + +// TODO deprecated ... use tus + +func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { + + node, err := fs.pw.NodeFromResource(ctx, ref) + if err != nil { + return err + } + + if node.ID == "" { + node.ID = uuid.New().String() + } + + nodePath := filepath.Join(fs.pw.Root, "nodes", node.ID) + + tmp, err := ioutil.TempFile(nodePath, "._reva_atomic_upload") + if err != nil { + return errors.Wrap(err, "ocisfs: error creating tmp fn at "+nodePath) + } + + _, err = io.Copy(tmp, r) + if err != nil { + return errors.Wrap(err, "ocisfs: error writing to tmp file "+tmp.Name()) + } + + // TODO move old content to version + //_ = os.RemoveAll(path.Join(nodePath, "content")) + + err = os.Rename(tmp.Name(), nodePath) + if err != nil { + return err + } + return fs.tp.Propagate(ctx, node) + +} + +// InitiateUpload returns an upload id that can be used for uploads with tus +// TODO read optional content for small files in this request +func (fs *ocisfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (uploadID string, err error) { + + log := appctx.GetLogger(ctx) + + var relative string // the internal path of the file node + + node, err := fs.pw.NodeFromResource(ctx, ref) + if err != nil { + return "", err + } + + relative, err = fs.pw.Path(ctx, node) + if err != nil { + return "", err + } + + info := tusd.FileInfo{ + MetaData: tusd.MetaData{ + "filename": filepath.Base(relative), + "dir": filepath.Dir(relative), + }, + Size: uploadLength, + } + + if metadata != nil && metadata["mtime"] != "" { + info.MetaData["mtime"] = metadata["mtime"] + } + + log.Debug().Interface("info", info).Interface("node", node).Interface("metadata", metadata).Msg("ocisfs: resolved filename") + + upload, err := fs.NewUpload(ctx, info) + if err != nil { + return "", err + } + + info, _ = upload.GetInfo(ctx) + + return info.ID, nil +} + +// UseIn tells the tus upload middleware which extensions it supports. +func (fs *ocisfs) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(fs) + composer.UseTerminater(fs) + composer.UseConcater(fs) + composer.UseLengthDeferrer(fs) +} + +// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol +// - the storage needs to implement NewUpload and GetUpload +// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload + +func (fs *ocisfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { + + log := appctx.GetLogger(ctx) + log.Debug().Interface("info", info).Msg("ocisfs: NewUpload") + + fn := info.MetaData["filename"] + if fn == "" { + return nil, errors.New("ocisfs: missing filename in metadata") + } + info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) + + dir := info.MetaData["dir"] + if dir == "" { + return nil, errors.New("ocisfs: missing dir in metadata") + } + info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) + + node, err := fs.pw.NodeFromPath(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) + if err != nil { + return nil, errors.Wrap(err, "ocisfs: error wrapping filename") + } + + log.Debug().Interface("info", info).Interface("node", node).Msg("ocisfs: resolved filename") + + info.ID = uuid.New().String() + + binPath, err := fs.getUploadPath(ctx, info.ID) + if err != nil { + return nil, errors.Wrap(err, "ocisfs: error resolving upload path") + } + usr := user.ContextMustGetUser(ctx) + info.Storage = map[string]string{ + "Type": "OCISStore", + "BinPath": binPath, + + "NodeId": node.ID, + "NodeParentId": node.ParentID, + "NodeName": node.Name, + + "Idp": usr.Id.Idp, + "UserId": usr.Id.OpaqueId, + "UserName": usr.Username, + + "LogLevel": log.GetLevel().String(), + } + // Create binary file in the upload folder with no content + log.Debug().Interface("info", info).Msg("ocisfs: built storage info") + file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return nil, err + } + defer file.Close() + + u := &fileUpload{ + info: info, + binPath: binPath, + infoPath: filepath.Join(fs.pw.Root, "uploads", info.ID+".info"), + fs: fs, + ctx: ctx, + } + + if !info.SizeIsDeferred && info.Size == 0 { + log.Debug().Interface("info", info).Msg("ocisfs: finishing upload for empty file") + // no need to create info file and finish directly + err := u.FinishUpload(ctx) + if err != nil { + return nil, err + } + return u, nil + } + + // writeInfo creates the file by itself if necessary + err = u.writeInfo() + if err != nil { + return nil, err + } + + return u, nil +} + +func (fs *ocisfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { + return filepath.Join(fs.pw.Root, "uploads", uploadID), nil +} + +// GetUpload returns the Upload for the given upload id +func (fs *ocisfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + infoPath := filepath.Join(fs.pw.Root, "uploads", id+".info") + + info := tusd.FileInfo{} + data, err := ioutil.ReadFile(infoPath) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + stat, err := os.Stat(info.Storage["BinPath"]) + if err != nil { + return nil, err + } + + info.Offset = stat.Size() + + u := &userpb.User{ + Id: &userpb.UserId{ + Idp: info.Storage["Idp"], + OpaqueId: info.Storage["UserId"], + }, + Username: info.Storage["UserName"], + } + + ctx = user.ContextSetUser(ctx, u) + // TODO configure the logger the same way ... store and add traceid in file info + + var opts []logger.Option + opts = append(opts, logger.WithLevel(info.Storage["LogLevel"])) + opts = append(opts, logger.WithWriter(os.Stderr, logger.ConsoleMode)) + l := logger.New(opts...) + + sub := l.With().Int("pid", os.Getpid()).Logger() + + ctx = appctx.WithLogger(ctx, &sub) + + return &fileUpload{ + info: info, + binPath: info.Storage["BinPath"], + infoPath: infoPath, + fs: fs, + ctx: ctx, + }, nil +} + +type fileUpload struct { + // info stores the current information about the upload + info tusd.FileInfo + // infoPath is the path to the .info file + infoPath string + // binPath is the path to the binary file (which has no extension) + binPath string + // only fs knows how to handle metadata and versions + fs *ocisfs + // a context with a user + // TODO add logger as well? + ctx context.Context +} + +// GetInfo returns the FileInfo +func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return upload.info, nil +} + +// WriteChunk writes the stream from the reader to the given offset of the upload +func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return 0, err + } + defer file.Close() + + n, err := io.Copy(file, src) + + // If the HTTP PATCH request gets interrupted in the middle (e.g. because + // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. + // However, for the ocis driver it's not important whether the stream has ended + // on purpose or accidentally. + if err != nil { + if err != io.ErrUnexpectedEOF { + return n, err + } + } + + upload.info.Offset += n + err = upload.writeInfo() // TODO info is written here ... we need to truncate in DiscardChunk + + return n, err +} + +// GetReader returns an io.Reader for the upload +func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) { + return os.Open(upload.binPath) +} + +// writeInfo updates the entire information. Everything will be overwritten. +func (upload *fileUpload) writeInfo() error { + data, err := json.Marshal(upload.info) + if err != nil { + return err + } + return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm) +} + +// FinishUpload finishes an upload and moves the file to the internal destination +func (upload *fileUpload) FinishUpload(ctx context.Context) (err error) { + + n := &Node{ + pw: upload.fs.pw, + ID: upload.info.Storage["NodeId"], + ParentID: upload.info.Storage["NodeParentId"], + Name: upload.info.Storage["NodeName"], + } + + if n.ID == "" { + n.ID = uuid.New().String() + } + targetPath := filepath.Join(upload.fs.pw.Root, "nodes", n.ID) + + // if target exists create new version + var fi os.FileInfo + if fi, err = os.Stat(targetPath); err == nil { + // versions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries + versionsPath := filepath.Join(upload.fs.pw.Root, "nodes", n.ID+".REV."+fi.ModTime().UTC().Format(time.RFC3339Nano)) + + if err = os.Rename(targetPath, versionsPath); err != nil { + log := appctx.GetLogger(upload.ctx) + log.Err(err).Interface("info", upload.info). + Str("binPath", upload.binPath). + Str("targetPath", targetPath). + Msg("ocisfs: could not create version") + return + } + } + + // now rename the upload to the target path + // TODO put uploads on the same underlying storage as the destination dir? + // TODO trigger a workflow as the final rename might eg involve antivirus scanning + if err = os.Rename(upload.binPath, targetPath); err != nil { + log := appctx.GetLogger(upload.ctx) + log.Err(err).Interface("info", upload.info). + Str("binPath", upload.binPath). + Str("targetPath", targetPath). + Msg("ocisfs: could not rename") + return + } + if n.pw.EnableHome { + if u, ok := user.ContextGetUser(upload.ctx); ok { + err = n.writeMetadata(u.Id) + } else { + log := appctx.GetLogger(upload.ctx) + log.Error().Msg("home support enabled but no user in context") + err = errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") + } + } else { + err = n.writeMetadata(nil) + } + if err != nil { + return + } + + // link child name to parent if it is new + childNameLink := filepath.Join(upload.fs.pw.Root, "nodes", n.ParentID, n.Name) + var link string + link, err = os.Readlink(childNameLink) + if err == nil && link != "../"+n.ID { + log.Err(err). + Interface("info", upload.info). + Interface("node", n). + Str("targetPath", targetPath). + Str("childNameLink", childNameLink). + Str("link", link). + Msg("ocisfs: child name link has wrong target id, repairing") + + if err = os.Remove(childNameLink); err != nil { + return errors.Wrap(err, "ocisfs: could not remove symlink child entry") + } + } + if os.IsNotExist(err) || link != "../"+n.ID { + if err = os.Symlink("../"+n.ID, childNameLink); err != nil { + return errors.Wrap(err, "ocisfs: could not symlink child entry") + } + } + + // only delete the upload if it was successfully written to the storage + if err = os.Remove(upload.infoPath); err != nil { + if !os.IsNotExist(err) { + log.Err(err).Interface("info", upload.info).Msg("ocisfs: could not delete upload info") + return + } + } + // use set arbitrary metadata? + /*if upload.info.MetaData["mtime"] != "" { + err := upload.fs.SetMtime(ctx, np, upload.info.MetaData["mtime"]) + if err != nil { + log.Err(err).Interface("info", upload.info).Msg("ocisfs: could not set mtime metadata") + return err + } + }*/ + + n.Exists = true + + return upload.fs.tp.Propagate(upload.ctx, n) +} + +// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination +// - the storage needs to implement AsTerminatableUpload +// - the upload needs to implement Terminate + +// AsTerminatableUpload returns a TerminatableUpload +func (fs *ocisfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { + return upload.(*fileUpload) +} + +// Terminate terminates the upload +func (upload *fileUpload) Terminate(ctx context.Context) error { + if err := os.Remove(upload.infoPath); err != nil { + if !os.IsNotExist(err) { + return err + } + } + if err := os.Remove(upload.binPath); err != nil { + if !os.IsNotExist(err) { + return err + } + } + return nil +} + +// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation +// - the storage needs to implement AsLengthDeclarableUpload +// - the upload needs to implement DeclareLength + +// AsLengthDeclarableUpload returns a LengthDeclarableUpload +func (fs *ocisfs) AsLengthDeclarableUpload(upload tusd.Upload) tusd.LengthDeclarableUpload { + return upload.(*fileUpload) +} + +// DeclareLength updates the upload length information +func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error { + upload.info.Size = length + upload.info.SizeIsDeferred = false + return upload.writeInfo() +} + +// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation +// - the storage needs to implement AsConcatableUpload +// - the upload needs to implement ConcatUploads + +// AsConcatableUpload returns a ConcatableUpload +func (fs *ocisfs) AsConcatableUpload(upload tusd.Upload) tusd.ConcatableUpload { + return upload.(*fileUpload) +} + +// ConcatUploads concatenates multiple uploads +func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Upload) (err error) { + file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return err + } + defer file.Close() + + for _, partialUpload := range uploads { + fileUpload := partialUpload.(*fileUpload) + + src, err := os.Open(fileUpload.binPath) + if err != nil { + return err + } + + if _, err := io.Copy(file, src); err != nil { + return err + } + } + + return +} diff --git a/pkg/storage/fs/owncloud/owncloud.go b/pkg/storage/fs/owncloud/owncloud.go index cb51617be4d..cdd8e028e1c 100644 --- a/pkg/storage/fs/owncloud/owncloud.go +++ b/pkg/storage/fs/owncloud/owncloud.go @@ -20,7 +20,6 @@ package owncloud import ( "context" - "encoding/csv" "fmt" "io" "io/ioutil" @@ -44,6 +43,7 @@ import ( "github.com/cs3org/reva/pkg/sharedconf" "github.com/cs3org/reva/pkg/storage" "github.com/cs3org/reva/pkg/storage/fs/registry" + "github.com/cs3org/reva/pkg/storage/utils/ace" "github.com/cs3org/reva/pkg/storage/utils/templates" "github.com/cs3org/reva/pkg/user" "github.com/gofrs/uuid" @@ -64,83 +64,6 @@ const ( // idAttribute is the name of the filesystem extended attribute that is used to store the uuid in idAttribute string = "user.oc.id" - // shares are persisted using extended attributes. We are going to mimic - // NFS4 ACLs, with one extended attribute per share, following Access - // Control Entries (ACEs). The following is taken from the nfs4_acl man page, - // see https://linux.die.net/man/5/nfs4_acl: - // the extended attributes will look like this - // "user.oc.acl.:::" - // - *type* will be limited to A for now - // A: Allow - allow *principal* to perform actions requiring *permissions* - // In the future we can use: - // U: aUdit - log any attempted access by principal which requires - // permissions. - // L: aLarm - generate a system alarm at any attempted access by - // principal which requires permissions - // D for deny is not recommended - // - *flags* for now empty or g for group, no inheritance yet - // - d directory-inherit - newly-created subdirectories will inherit the - // ACE. - // - f file-inherit - newly-created files will inherit the ACE, minus its - // inheritance flags. Newly-created subdirectories - // will inherit the ACE; if directory-inherit is not - // also specified in the parent ACE, inherit-only will - // be added to the inherited ACE. - // - n no-propagate-inherit - newly-created subdirectories will inherit - // the ACE, minus its inheritance flags. - // - i inherit-only - the ACE is not considered in permissions checks, - // but it is heritable; however, the inherit-only - // flag is stripped from inherited ACEs. - // - *principal* a named user, group or special principal - // - the oidc sub@iss maps nicely to this - // - 'OWNER@', 'GROUP@', and 'EVERYONE@', which are, respectively, analogous to the POSIX user/group/other - // - *permissions* - // - r read-data (files) / list-directory (directories) - // - w write-data (files) / create-file (directories) - // - a append-data (files) / create-subdirectory (directories) - // - x execute (files) / change-directory (directories) - // - d delete - delete the file/directory. Some servers will allow a delete to occur if either this permission is set in the file/directory or if the delete-child permission is set in its parent directory. - // - D delete-child - remove a file or subdirectory from within the given directory (directories only) - // - t read-attributes - read the attributes of the file/directory. - // - T write-attributes - write the attributes of the file/directory. - // - n read-named-attributes - read the named attributes of the file/directory. - // - N write-named-attributes - write the named attributes of the file/directory. - // - c read-ACL - read the file/directory NFSv4 ACL. - // - C write-ACL - write the file/directory NFSv4 ACL. - // - o write-owner - change ownership of the file/directory. - // - y synchronize - allow clients to use synchronous I/O with the server. - // TODO implement OWNER@ as "user.oc.acl.A::OWNER@:rwaDxtTnNcCy" - // attribute names are limited to 255 chars by the linux kernel vfs, values to 64 kb - // ext3 extended attributes must fit inside a single filesystem block ... 4096 bytes - // that leaves us with "user.oc.acl.A::someonewithaslightlylongersubject@whateverissuer:rwaDxtTnNcCy" ~80 chars - // 4096/80 = 51 shares ... with luck we might move the actual permissions to the value, saving ~15 chars - // 4096/64 = 64 shares ... still meh ... we can do better by using ints instead of strings for principals - // "user.oc.acl.u:100000" is pretty neat, but we can still do better: base64 encode the int - // "user.oc.acl.u:6Jqg" but base64 always has at least 4 chars, maybe hex is better for smaller numbers - // well use 4 chars in addition to the ace: "user.oc.acl.u:////" = 65535 -> 18 chars - // 4096/18 = 227 shares - // still ... ext attrs for this are not infinite scale ... - // so .. attach shares via fileid. - // /metadata//shares, similar to /files - // /metadata//shares/u///A:fdi:rwaDxtTnNcCy permissions as filename to keep them in the stat cache? - // - // whatever ... 50 shares is good enough. If more is needed we can delegate to the metadata - // if "user.oc.acl.M" is present look inside the metadata app. - // - if we cannot set an ace we might get an io error. - // in that case convert all shares to metadata and try to set "user.oc.acl.m" - // - // what about metadata like share creator, share time, expiry? - // - creator is same as owner, but can be set - // - share date, or abbreviated st is a unix timestamp - // - expiry is a unix timestamp - // - can be put inside the value - // - we need to reorder the fields: - // "user.oc.acl.:" -> "kv:t=:f=:p=:st=:c=:e=:pw=:n=" - // "user.oc.acl.:" -> "v1::::::::" - // or the first byte determines the format - // 0x00 = key value - // 0x01 = v1 ... - // // SharePrefix is the prefix for sharing related extended attributes sharePrefix string = "user.oc.acl." trashOriginPrefix string = "user.oc.o" @@ -155,14 +78,15 @@ func init() { } type config struct { - DataDirectory string `mapstructure:"datadirectory"` - UploadInfoDir string `mapstructure:"upload_info_dir"` - ShareDirectory string `mapstructure:"sharedirectory"` - UserLayout string `mapstructure:"user_layout"` - Redis string `mapstructure:"redis"` - EnableHome bool `mapstructure:"enable_home"` - Scan bool `mapstructure:"scan"` - UserProviderEndpoint string `mapstructure:"userprovidersvc"` + DataDirectory string `mapstructure:"datadirectory"` + UploadInfoDir string `mapstructure:"upload_info_dir"` + DeprecatedShareDirectory string `mapstructure:"sharedirectory"` + ShareFolder string `mapstructure:"share_folder"` + UserLayout string `mapstructure:"user_layout"` + Redis string `mapstructure:"redis"` + EnableHome bool `mapstructure:"enable_home"` + Scan bool `mapstructure:"scan"` + UserProviderEndpoint string `mapstructure:"userprovidersvc"` } func parseConfig(m map[string]interface{}) (*config, error) { @@ -184,9 +108,16 @@ func (c *config) init(m map[string]interface{}) { if c.UploadInfoDir == "" { c.UploadInfoDir = "/var/tmp/reva/uploadinfo" } - if c.ShareDirectory == "" { - c.ShareDirectory = "/Shares" + // fallback for old config + if c.DeprecatedShareDirectory != "" { + c.ShareFolder = c.DeprecatedShareDirectory } + if c.ShareFolder == "" { + c.ShareFolder = "/Shares" + } + // ensure share folder always starts with slash + c.ShareFolder = filepath.Join("/", c.ShareFolder) + // default to scanning if not configured if _, ok := m["scan"]; !ok { c.Scan = true @@ -750,209 +681,36 @@ func (fs *ocfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provid return errors.Wrap(err, "ocfs: error resolving reference") } - e, err := fs.getACE(g) - if err != nil { - return err - } - - var attr string - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - attr = sharePrefix + "g:" + e.Principal - } else { - attr = sharePrefix + "u:" + e.Principal - } - - if err := xattr.Set(np, attr, getValue(e)); err != nil { + e := ace.FromGrant(g) + principal, value := e.Marshal() + if err := xattr.Set(np, sharePrefix+principal, value); err != nil { return err } return fs.propagate(ctx, np) } -func getValue(e *ace) []byte { - // first byte will be replaced after converting to byte array - val := fmt.Sprintf("_t=%s:f=%s:p=%s", e.Type, e.Flags, e.Permissions) - b := []byte(val) - b[0] = 0 // indicate key value - return b -} - -func getACEPerm(set *provider.ResourcePermissions) (string, error) { - var b strings.Builder - - if set.Stat || set.InitiateFileDownload || set.ListContainer { - b.WriteString("r") - } - if set.InitiateFileUpload || set.Move { - b.WriteString("w") - } - if set.CreateContainer { - b.WriteString("a") - } - if set.Delete { - b.WriteString("d") - } - - // sharing - if set.AddGrant || set.RemoveGrant || set.UpdateGrant { - b.WriteString("C") - } - if set.ListGrants { - b.WriteString("c") - } - - // trash - if set.ListRecycle { - b.WriteString("u") - } - if set.RestoreRecycleItem { - b.WriteString("U") - } - if set.PurgeRecycle { - b.WriteString("P") - } - - // versions - if set.ListFileVersions { - b.WriteString("v") - } - if set.RestoreFileVersion { - b.WriteString("V") - } - - // quota - if set.GetQuota { - b.WriteString("q") - } - // TODO set quota permission? - // TODO GetPath - return b.String(), nil -} - -func (fs *ocfs) getACE(g *provider.Grant) (*ace, error) { - permissions, err := getACEPerm(g.Permissions) - if err != nil { - return nil, err - } - e := &ace{ - Principal: g.Grantee.Id.OpaqueId, - Permissions: permissions, - // TODO creator ... - Type: "A", - } - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - e.Flags = "g" - } - return e, nil -} - -type ace struct { - //NFSv4 acls - Type string // t - Flags string // f - Principal string // im key - Permissions string // p - - // sharing specific - ShareTime int // s - Creator string // c - Expires int // e - Password string // w passWord TODO h = hash - Label string // l -} - -func unmarshalACE(v []byte) (*ace, error) { - // first byte indicates type of value - switch v[0] { - case 0: // = ':' separated key=value pairs - s := string(v[1:]) - return unmarshalKV(s) - default: - return nil, fmt.Errorf("unknown ace encoding") - } -} - -func unmarshalKV(s string) (*ace, error) { - e := &ace{} - r := csv.NewReader(strings.NewReader(s)) - r.Comma = ':' - r.Comment = 0 - r.FieldsPerRecord = -1 - r.LazyQuotes = false - r.TrimLeadingSpace = false - records, err := r.ReadAll() - if err != nil { - return nil, err - } - if len(records) != 1 { - return nil, fmt.Errorf("more than one row of ace kvs") - } - for i := range records[0] { - kv := strings.Split(records[0][i], "=") - switch kv[0] { - case "t": - e.Type = kv[1] - case "f": - e.Flags = kv[1] - case "p": - e.Permissions = kv[1] - case "s": - v, err := strconv.Atoi(kv[1]) - if err != nil { - return nil, err - } - e.ShareTime = v - case "c": - e.Creator = kv[1] - case "e": - v, err := strconv.Atoi(kv[1]) - if err != nil { - return nil, err - } - e.Expires = v - case "w": - e.Password = kv[1] - case "l": - e.Label = kv[1] - // TODO default ... log unknown keys? or add as opaque? hm we need that for tagged shares ... - } - } - return e, nil -} - -// Parse parses an acl string with the given delimiter (LongTextForm or ShortTextForm) -func getACEs(ctx context.Context, fsfn string, attrs []string) (entries []*ace, err error) { +// extractACEsFromAttrs reads ACEs in the list of attrs from the file +func extractACEsFromAttrs(ctx context.Context, fsfn string, attrs []string) (entries []*ace.ACE) { log := appctx.GetLogger(ctx) - entries = []*ace{} + entries = []*ace.ACE{} for i := range attrs { if strings.HasPrefix(attrs[i], sharePrefix) { - principal := attrs[i][len(sharePrefix):] var value []byte + var err error if value, err = xattr.Get(fsfn, attrs[i]); err != nil { log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute") continue } - var e *ace - if e, err = unmarshalACE(value); err != nil { - log.Error().Err(err).Str("attr", attrs[i]).Msg("could unmarshal ace") + var e *ace.ACE + principal := attrs[i][len(sharePrefix):] + if e, err = ace.Unmarshal(principal, value); err != nil { + log.Error().Err(err).Str("principal", principal).Str("attr", attrs[i]).Msg("could not unmarshal ace") continue } - e.Principal = principal[2:] - // check consistency of Flags and principal type - if strings.Contains(e.Flags, "g") { - if principal[:1] != "g" { - log.Error().Str("attr", attrs[i]).Interface("ace", e).Msg("inconsistent ace: expected group") - continue - } - } else { - if principal[:1] != "u" { - log.Error().Str("attr", attrs[i]).Interface("ace", e).Msg("inconsistent ace: expected user") - continue - } - } entries = append(entries, e) } } - return entries, nil + return } func (fs *ocfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { @@ -968,105 +726,17 @@ func (fs *ocfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants } log.Debug().Interface("attrs", attrs).Msg("read attributes") - // filter attributes - var aces []*ace - if aces, err = getACEs(ctx, np, attrs); err != nil { - log.Error().Err(err).Msg("error getting aces") - return nil, err - } + + aces := extractACEsFromAttrs(ctx, np, attrs) grants = make([]*provider.Grant, 0, len(aces)) for i := range aces { - grantee := &provider.Grantee{ - // TODO lookup uid from principal - Id: &userpb.UserId{OpaqueId: aces[i].Principal}, - Type: fs.getGranteeType(aces[i]), - } - grants = append(grants, &provider.Grant{ - Grantee: grantee, - Permissions: fs.getGrantPermissionSet(aces[i].Permissions), - }) + grants = append(grants, aces[i].Grant()) } return grants, nil } -func (fs *ocfs) getGranteeType(e *ace) provider.GranteeType { - if strings.Contains(e.Flags, "g") { - return provider.GranteeType_GRANTEE_TYPE_GROUP - } - return provider.GranteeType_GRANTEE_TYPE_USER -} - -func (fs *ocfs) getGrantPermissionSet(mode string) *provider.ResourcePermissions { - p := &provider.ResourcePermissions{} - // r - if strings.Contains(mode, "r") { - p.Stat = true - p.InitiateFileDownload = true - p.ListContainer = true - } - // w - if strings.Contains(mode, "w") { - p.InitiateFileUpload = true - if p.InitiateFileDownload { - p.Move = true - } - } - //a - if strings.Contains(mode, "a") { - // TODO append data to file permission? - p.CreateContainer = true - } - //x - //if strings.Contains(mode, "x") { - // TODO execute file permission? - // TODO change directory permission? - //} - //d - if strings.Contains(mode, "d") { - p.Delete = true - } - //D ? - - // sharing - if strings.Contains(mode, "C") { - p.AddGrant = true - p.RemoveGrant = true - p.UpdateGrant = true - } - if strings.Contains(mode, "c") { - p.ListGrants = true - } - - // trash - if strings.Contains(mode, "u") { // u = undelete - p.ListRecycle = true - } - if strings.Contains(mode, "U") { - p.RestoreRecycleItem = true - } - if strings.Contains(mode, "P") { - p.PurgeRecycle = true - } - - // versions - if strings.Contains(mode, "v") { - p.ListFileVersions = true - } - if strings.Contains(mode, "V") { - p.RestoreFileVersion = true - } - - // ? - // TODO GetPath - if strings.Contains(mode, "q") { - p.GetQuota = true - } - // TODO set quota permission? - return p -} - func (fs *ocfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { var np string @@ -1142,11 +812,11 @@ func (fs *ocfs) CreateDir(ctx context.Context, fn string) (err error) { } func (fs *ocfs) isShareFolderChild(p string) bool { - return strings.HasPrefix(p, fs.c.ShareDirectory) + return strings.HasPrefix(p, fs.c.ShareFolder) } func (fs *ocfs) isShareFolderRoot(p string) bool { - return p == fs.c.ShareDirectory + return p == fs.c.ShareFolder } func (fs *ocfs) CreateReference(ctx context.Context, p string, targetURI *url.URL) error { diff --git a/pkg/storage/utils/ace/ace.go b/pkg/storage/utils/ace/ace.go new file mode 100644 index 00000000000..14154bf4680 --- /dev/null +++ b/pkg/storage/utils/ace/ace.go @@ -0,0 +1,364 @@ +// Copyright 2018-2020 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package ace + +import ( + "encoding/csv" + "fmt" + "strconv" + "strings" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" +) + +// ACE represents an Access Control Entry, mimicing NFSv4 ACLs +// +// The following is taken from the nfs4_acl man page, +// see https://linux.die.net/man/5/nfs4_acl: +// the extended attributes will look like this +// "user.oc.acl.:::" +// - *type* will be limited to A for now +// A: Allow - allow *principal* to perform actions requiring *permissions* +// In the future we can use: +// U: aUdit - log any attempted access by principal which requires +// permissions. +// L: aLarm - generate a system alarm at any attempted access by +// principal which requires permissions +// D for deny is not recommended +// - *flags* for now empty or g for group, no inheritance yet +// - d directory-inherit - newly-created subdirectories will inherit the +// ACE. +// - f file-inherit - newly-created files will inherit the ACE, minus its +// inheritance flags. Newly-created subdirectories +// will inherit the ACE; if directory-inherit is not +// also specified in the parent ACE, inherit-only will +// be added to the inherited ACE. +// - n no-propagate-inherit - newly-created subdirectories will inherit +// the ACE, minus its inheritance flags. +// - i inherit-only - the ACE is not considered in permissions checks, +// but it is heritable; however, the inherit-only +// flag is stripped from inherited ACEs. +// - *principal* a named user, group or special principal +// - the oidc sub@iss maps nicely to this +// - 'OWNER@', 'GROUP@', and 'EVERYONE@', which are, respectively, analogous to the POSIX user/group/other +// - *permissions* +// - r read-data (files) / list-directory (directories) +// - w write-data (files) / create-file (directories) +// - a append-data (files) / create-subdirectory (directories) +// - x execute (files) / change-directory (directories) +// - d delete - delete the file/directory. Some servers will allow a delete to occur if either this permission is set in the file/directory or if the delete-child permission is set in its parent directory. +// - D delete-child - remove a file or subdirectory from within the given directory (directories only) +// - t read-attributes - read the attributes of the file/directory. +// - T write-attributes - write the attributes of the file/directory. +// - n read-named-attributes - read the named attributes of the file/directory. +// - N write-named-attributes - write the named attributes of the file/directory. +// - c read-ACL - read the file/directory NFSv4 ACL. +// - C write-ACL - write the file/directory NFSv4 ACL. +// - o write-owner - change ownership of the file/directory. +// - y synchronize - allow clients to use synchronous I/O with the server. +// TODO implement OWNER@ as "user.oc.acl.A::OWNER@:rwaDxtTnNcCy" +// attribute names are limited to 255 chars by the linux kernel vfs, values to 64 kb +// ext3 extended attributes must fit inside a single filesystem block ... 4096 bytes +// that leaves us with "user.oc.acl.A::someonewithaslightlylongersubject@whateverissuer:rwaDxtTnNcCy" ~80 chars +// 4096/80 = 51 shares ... with luck we might move the actual permissions to the value, saving ~15 chars +// 4096/64 = 64 shares ... still meh ... we can do better by using ints instead of strings for principals +// "user.oc.acl.u:100000" is pretty neat, but we can still do better: base64 encode the int +// "user.oc.acl.u:6Jqg" but base64 always has at least 4 chars, maybe hex is better for smaller numbers +// well use 4 chars in addition to the ace: "user.oc.acl.u:////" = 65535 -> 18 chars +// 4096/18 = 227 shares +// still ... ext attrs for this are not infinite scale ... +// so .. attach shares via fileid. +// /metadata//shares, similar to /files +// /metadata//shares/u///A:fdi:rwaDxtTnNcCy permissions as filename to keep them in the stat cache? +// +// whatever ... 50 shares is good enough. If more is needed we can delegate to the metadata +// if "user.oc.acl.M" is present look inside the metadata app. +// - if we cannot set an ace we might get an io error. +// in that case convert all shares to metadata and try to set "user.oc.acl.m" +// +// what about metadata like share creator, share time, expiry? +// - creator is same as owner, but can be set +// - share date, or abbreviated st is a unix timestamp +// - expiry is a unix timestamp +// - can be put inside the value +// - we need to reorder the fields: +// "user.oc.acl.:" -> "kv:t=:f=:p=:st=:c=:e=:pw=:n=" +// "user.oc.acl.:" -> "v1::::::::" +// or the first byte determines the format +// 0x00 = key value +// 0x01 = v1 ... +type ACE struct { + //NFSv4 acls + _type string // t + flags string // f + principal string // im key + permissions string // p + + // sharing specific + shareTime int // s + creator string // c + expires int // e + password string // w passWord TODO h = hash + label string // l +} + +// FromGrant creates an ACE from a CS3 grant +func FromGrant(g *provider.Grant) *ACE { + e := &ACE{ + _type: "A", + permissions: getACEPerm(g.Permissions), + // TODO creator ... + } + if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { + e.flags = "g" + e.principal = "g:" + g.Grantee.Id.OpaqueId + } else { + e.principal = "u:" + g.Grantee.Id.OpaqueId + } + return e +} + +// Principal returns the principal of the ACE, eg. `u:` or `g:` +func (e *ACE) Principal() string { + return e.principal +} + +// Marshal renders a principal and byte[] that can be used to persist the ACE as an extended attribute +func (e *ACE) Marshal() (string, []byte) { + // first byte will be replaced after converting to byte array + val := fmt.Sprintf("_t=%s:f=%s:p=%s", e._type, e.flags, e.permissions) + b := []byte(val) + b[0] = 0 // indicate key value + return e.principal, b +} + +// Unmarshal parses a principal string and byte[] into an ACE +func Unmarshal(principal string, v []byte) (e *ACE, err error) { + // first byte indicates type of value + switch v[0] { + case 0: // = ':' separated key=value pairs + s := string(v[1:]) + if e, err = unmarshalKV(s); err == nil { + e.principal = principal + } + // check consistency of Flags and principal type + if strings.Contains(e.flags, "g") { + if principal[:1] != "g" { + return nil, fmt.Errorf("inconsistent ace: expected group") + } + } else { + if principal[:1] != "u" { + return nil, fmt.Errorf("inconsistent ace: expected user") + } + } + default: + return nil, fmt.Errorf("unknown ace encoding") + } + return +} + +// Grant returns a CS3 grant +func (e *ACE) Grant() *provider.Grant { + return &provider.Grant{ + Grantee: &provider.Grantee{ + Id: &userpb.UserId{OpaqueId: e.principal}, + Type: e.granteeType(), + }, + Permissions: e.grantPermissionSet(), + } +} + +// granteeType returns the CS3 grantee type +func (e *ACE) granteeType() provider.GranteeType { + if strings.Contains(e.flags, "g") { + return provider.GranteeType_GRANTEE_TYPE_GROUP + } + return provider.GranteeType_GRANTEE_TYPE_USER +} + +// grantPermissionSet returns the set of CS3 resource permissions representing the ACE +func (e *ACE) grantPermissionSet() *provider.ResourcePermissions { + p := &provider.ResourcePermissions{} + // r + if strings.Contains(e.permissions, "r") { + p.Stat = true + p.InitiateFileDownload = true + p.ListContainer = true + } + // w + if strings.Contains(e.permissions, "w") { + p.InitiateFileUpload = true + if p.InitiateFileDownload { + p.Move = true + } + } + //a + if strings.Contains(e.permissions, "a") { + // TODO append data to file permission? + p.CreateContainer = true + } + //x + //if strings.Contains(e.Permissions, "x") { + // TODO execute file permission? + // TODO change directory permission? + //} + //d + if strings.Contains(e.permissions, "d") { + p.Delete = true + } + //D ? + + // sharing + if strings.Contains(e.permissions, "C") { + p.AddGrant = true + p.RemoveGrant = true + p.UpdateGrant = true + } + if strings.Contains(e.permissions, "c") { + p.ListGrants = true + } + + // trash + if strings.Contains(e.permissions, "u") { // u = undelete + p.ListRecycle = true + } + if strings.Contains(e.permissions, "U") { + p.RestoreRecycleItem = true + } + if strings.Contains(e.permissions, "P") { + p.PurgeRecycle = true + } + + // versions + if strings.Contains(e.permissions, "v") { + p.ListFileVersions = true + } + if strings.Contains(e.permissions, "V") { + p.RestoreFileVersion = true + } + + // ? + // TODO GetPath + if strings.Contains(e.permissions, "q") { + p.GetQuota = true + } + // TODO set quota permission? + return p +} + +func unmarshalKV(s string) (*ACE, error) { + e := &ACE{} + r := csv.NewReader(strings.NewReader(s)) + r.Comma = ':' + r.Comment = 0 + r.FieldsPerRecord = -1 + r.LazyQuotes = false + r.TrimLeadingSpace = false + records, err := r.ReadAll() + if err != nil { + return nil, err + } + if len(records) != 1 { + return nil, fmt.Errorf("more than one row of ace kvs") + } + for i := range records[0] { + kv := strings.Split(records[0][i], "=") + switch kv[0] { + case "t": + e._type = kv[1] + case "f": + e.flags = kv[1] + case "p": + e.permissions = kv[1] + case "s": + v, err := strconv.Atoi(kv[1]) + if err != nil { + return nil, err + } + e.shareTime = v + case "c": + e.creator = kv[1] + case "e": + v, err := strconv.Atoi(kv[1]) + if err != nil { + return nil, err + } + e.expires = v + case "w": + e.password = kv[1] + case "l": + e.label = kv[1] + // TODO default ... log unknown keys? or add as opaque? hm we need that for tagged shares ... + } + } + return e, nil +} + +func getACEPerm(set *provider.ResourcePermissions) string { + var b strings.Builder + + if set.Stat || set.InitiateFileDownload || set.ListContainer { + b.WriteString("r") + } + if set.InitiateFileUpload || set.Move { + b.WriteString("w") + } + if set.CreateContainer { + b.WriteString("a") + } + if set.Delete { + b.WriteString("d") + } + + // sharing + if set.AddGrant || set.RemoveGrant || set.UpdateGrant { + b.WriteString("C") + } + if set.ListGrants { + b.WriteString("c") + } + + // trash + if set.ListRecycle { + b.WriteString("u") + } + if set.RestoreRecycleItem { + b.WriteString("U") + } + if set.PurgeRecycle { + b.WriteString("P") + } + + // versions + if set.ListFileVersions { + b.WriteString("v") + } + if set.RestoreFileVersion { + b.WriteString("V") + } + + // quota + if set.GetQuota { + b.WriteString("q") + } + // TODO set quota permission? + // TODO GetPath + return b.String() +}