S3 blobstore driver

go.mod

@@ -33,6 +33,8 @@ require (
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.3.3
+	github.com/onsi/ginkgo v1.7.0
+	github.com/onsi/gomega v1.4.3
 	github.com/ory/fosite v0.35.1
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/xattr v0.4.2

go.sum

@@ -211,6 +211,7 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.0 h1:gvV6jG9dTgFEncxo+AF7PH6MZXi/vZl25owA/8Dg8Wo=
 github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
@@ -294,7 +295,9 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn
 github.com/oleiade/reflections v1.0.0 h1:0ir4pc6v8/PJ0yw5AEtMddfXpWBXg9cnG7SgSoJuCgY=
 github.com/oleiade/reflections v1.0.0/go.mod h1:RbATFBbKYkVdqmSFtx13Bb/tVhR0lgOBXunWTZKeL4w=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/ory/fosite v0.35.1 h1:mGPcwVGwHA7Yy9wr/7LDps6BEXyavL32NxizL9eH53Q=
@@ -385,6 +388,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -657,6 +661,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
@@ -665,6 +670,7 @@ gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w=
 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/storage/fs/loader/loader.go

@@ -29,5 +29,6 @@ import (
 	_ "github.com/cs3org/reva/pkg/storage/fs/ocis"
 	_ "github.com/cs3org/reva/pkg/storage/fs/owncloud"
 	_ "github.com/cs3org/reva/pkg/storage/fs/s3"
+	_ "github.com/cs3org/reva/pkg/storage/fs/s3ng"
 	// Add your own here
 )

pkg/storage/fs/s3ng/blobstore/blobstore.go

@@ -0,0 +1,66 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package blobstore
+
+import (
+	"io"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/pkg/errors"
+)
+
+// Blobstore provides an interface to an s3 compatible blobstore
+type Blobstore struct {
+	uploader *s3manager.Uploader
+	bucket   string
+}
+
+// New returns a new Blobstore
+func New(endpoint, region, bucket, accessKey, secretKey string) (*Blobstore, error) {
+	sess, err := session.NewSession(&aws.Config{
+		Endpoint:         aws.String(endpoint),
+		Region:           aws.String(region),
+		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to setup s3 session")
+	}
+	uploader := s3manager.NewUploader(sess)
+
+	return &Blobstore{
+		uploader: uploader,
+	}, nil
+}
+
+// Upload stores some data in the blobstore under the given key
+func (bs *Blobstore) Upload(key string, reader io.Reader) error {
+	_, err := bs.uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(bs.bucket),
+		Key:    aws.String(key),
+		Body:   reader,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "could not store object: %s", key)
+	}
+	return nil
+}

pkg/storage/fs/s3ng/grants.go

@@ -0,0 +1,167 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package s3ng
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/cs3org/reva/pkg/appctx"
+	"github.com/cs3org/reva/pkg/errtypes"
+	"github.com/cs3org/reva/pkg/storage/utils/ace"
+	"github.com/pkg/xattr"
+)
+
+func (fs *s3ngfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) {
+	log := appctx.GetLogger(ctx)
+	log.Debug().Interface("ref", ref).Interface("grant", g).Msg("AddGrant()")
+	var node *Node
+	if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil {
+		return
+	}
+	if !node.Exists {
+		err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name))
+		return
+	}
+
+	ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool {
+		// TODO remove AddGrant or UpdateGrant grant from CS3 api, redundant? tracked in https://github.com/cs3org/cs3apis/issues/92
+		return rp.AddGrant || rp.UpdateGrant
+	})
+	switch {
+	case err != nil:
+		return errtypes.InternalError(err.Error())
+	case !ok:
+		return errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name))
+	}
+
+	np := fs.lu.toInternalPath(node.ID)
+	e := ace.FromGrant(g)
+	principal, value := e.Marshal()
+	if err := xattr.Set(np, grantPrefix+principal, value); err != nil {
+		return err
+	}
+	return fs.tp.Propagate(ctx, node)
+}
+
+func (fs *s3ngfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) {
+	var node *Node
+	if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil {
+		return
+	}
+	if !node.Exists {
+		err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name))
+		return
+	}
+
+	ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool {
+		return rp.ListGrants
+	})
+	switch {
+	case err != nil:
+		return nil, errtypes.InternalError(err.Error())
+	case !ok:
+		return nil, errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name))
+	}
+
+	log := appctx.GetLogger(ctx)
+	np := fs.lu.toInternalPath(node.ID)
+	var attrs []string
+	if attrs, err = xattr.List(np); err != nil {
+		log.Error().Err(err).Msg("error listing attributes")
+		return nil, err
+	}
+
+	log.Debug().Interface("attrs", attrs).Msg("read attributes")
+
+	aces := extractACEsFromAttrs(ctx, np, attrs)
+
+	grants = make([]*provider.Grant, 0, len(aces))
+	for i := range aces {
+		grants = append(grants, aces[i].Grant())
+	}
+
+	return grants, nil
+}
+
+func (fs *s3ngfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) {
+	var node *Node
+	if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil {
+		return
+	}
+	if !node.Exists {
+		err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name))
+		return
+	}
+
+	ok, err := fs.p.HasPermission(ctx, node, func(rp *provider.ResourcePermissions) bool {
+		return rp.RemoveGrant
+	})
+	switch {
+	case err != nil:
+		return errtypes.InternalError(err.Error())
+	case !ok:
+		return errtypes.PermissionDenied(filepath.Join(node.ParentID, node.Name))
+	}
+
+	var attr string
+	if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP {
+		attr = grantPrefix + _groupAcePrefix + g.Grantee.Id.OpaqueId
+	} else {
+		attr = grantPrefix + _userAcePrefix + g.Grantee.Id.OpaqueId
+	}
+
+	np := fs.lu.toInternalPath(node.ID)
+	if err = xattr.Remove(np, attr); err != nil {
+		return
+	}
+
+	return fs.tp.Propagate(ctx, node)
+}
+
+func (fs *s3ngfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error {
+	// TODO remove AddGrant or UpdateGrant grant from CS3 api, redundant? tracked in https://github.com/cs3org/cs3apis/issues/92
+	return fs.AddGrant(ctx, ref, g)
+}
+
+// extractACEsFromAttrs reads ACEs in the list of attrs from the node
+func extractACEsFromAttrs(ctx context.Context, fsfn string, attrs []string) (entries []*ace.ACE) {
+	log := appctx.GetLogger(ctx)
+	entries = []*ace.ACE{}
+	for i := range attrs {
+		if strings.HasPrefix(attrs[i], grantPrefix) {
+			var value []byte
+			var err error
+			if value, err = xattr.Get(fsfn, attrs[i]); err != nil {
+				log.Error().Err(err).Str("attr", attrs[i]).Msg("could not read attribute")
+				continue
+			}
+			var e *ace.ACE
+			principal := attrs[i][len(grantPrefix):]
+			if e, err = ace.Unmarshal(principal, value); err != nil {
+				log.Error().Err(err).Str("principal", principal).Str("attr", attrs[i]).Msg("could not unmarshal ace")
+				continue
+			}
+			entries = append(entries, e)
+		}
+	}
+	return
+}

pkg/storage/fs/s3ng/interfaces.go

@@ -0,0 +1,64 @@
+// Copyright 2018-2021 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package s3ng
+
+import (
+	"context"
+	"os"
+
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+)
+
+// TODO the different aspects of a storage: Tree, Lookup and Permissions should be able to be reusable
+// Below is a start of Interfaces that needs to be worked out further
+
+// TreePersistence is used to manage a tree hierarchy
+type TreePersistence interface {
+	GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error)
+	GetMD(ctx context.Context, node *Node) (os.FileInfo, error)
+	ListFolder(ctx context.Context, node *Node) ([]*Node, error)
+	//CreateHome(owner *userpb.UserId) (n *Node, err error)
+	CreateDir(ctx context.Context, node *Node) (err error)
+	//CreateReference(ctx context.Context, node *Node, targetURI *url.URL) error
+	Move(ctx context.Context, oldNode *Node, newNode *Node) (err error)
+	Delete(ctx context.Context, node *Node) (err error)
+
+	Propagate(ctx context.Context, node *Node) (err error)
+}
+
+// Lookup is used to encapsulate path transformations
+/*
+type Lookup interface {
+	NodeFromResource(ctx context.Context, ref *provider.Reference) (node *Node, err error)
+	NodeFromID(ctx context.Context, id *provider.ResourceId) (node *Node, err error)
+	NodeFromPath(ctx context.Context, fn string) (node *Node, err error)
+	Path(ctx context.Context, node *Node) (path string, err error)
+
+	// HomeNode returns the currently logged in users home node
+	// requires EnableHome to be true
+	HomeNode(ctx context.Context) (node *Node, err error)
+
+	// RootNode returns the storage root node
+	RootNode(ctx context.Context) (node *Node, err error)
+
+	// HomeOrRootNode returns the users home node when home support is enabled.
+	// it returns the storages root node otherwise
+	HomeOrRootNode(ctx context.Context) (node *Node, err error)
+}
+*/