Merge branch 'master' into cernbox

.drone.env

@@ -1,4 +1,4 @@
 # The test runner source for API tests
-APITESTS_COMMITID=76cc388e4546d4e588515c6c7d624829a674163a
+APITESTS_COMMITID=4076bc71e63b4dd4d7931e1ad085ee214f137e38
 APITESTS_BRANCH=master
 APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git

.drone.star

@@ -119,7 +119,7 @@ def ocisIntegrationTest():
                     "REVA_LDAP_HOSTNAME": "ldap",
                     "TEST_REVA": "true",
                     "SEND_SCENARIO_LINE_REFERENCES": "true",
-                    "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva",
+                    "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster",
                     "DIVIDE_INTO_NUM_PARTS": 6,
                     "RUN_PART": 2,
                     "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md",
@@ -190,7 +190,7 @@ def s3ngIntegrationTests():
                             "REVA_LDAP_HOSTNAME": "ldap",
                             "TEST_REVA": "true",
                             "SEND_SCENARIO_LINE_REFERENCES": "true",
-                            "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva",
+                            "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster",
                             "DIVIDE_INTO_NUM_PARTS": parallelRuns,
                             "RUN_PART": runPart,
                             "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md",

changelog/unreleased/apps-viewmode.md

@@ -0,0 +1,6 @@
+Bugfix: Apps: fixed viewMode resolution
+
+Currently, the viewMode passed on /app/open is taken without validating
+the actual user's permissions. This PR fixes this.
+
+https://github.com/cs3org/reva/pull/3805

changelog/unreleased/datatx-new-ocm-impl.md

@@ -0,0 +1,4 @@
+Enhancement: Update data transfers for current OCM shares implementation
+
+https://github.com/cs3org/reva/pull/3847
+https://github.com/cs3org/reva/issues/3846

changelog/unreleased/fix-3747.md

@@ -0,0 +1,3 @@
+Bugfix: Fix persisting updates of received shares in json driver
+
+https://github.com/cs3org/reva/pull/3749

changelog/unreleased/fix-upload-lw-accounts-single-file-share.md

@@ -0,0 +1,3 @@
+Bugfix: Fix upload in a single file share for lightweight accounts
+
+https://github.com/cs3org/reva/pull/3838

changelog/unreleased/mentix-prodflags.md

@@ -0,0 +1,5 @@
+Enhancement: New metadata flags
+
+Several new flags, like site infrastructure and service status, are now gathered and exposed by Mentix.
+
+https://github.com/cs3org/reva/pull/3750

changelog/unreleased/multiple-issuers-oidc.md

@@ -0,0 +1,9 @@
+Enhancement: Support multiple issuer in OIDC auth driver
+
+The OIDC auth driver supports now multiple issuers. Users of
+external providers are then mapped to a local user by a 
+mapping files. Only the main issuer (defined in the config
+with `issuer`) and the ones defined in the mapping are
+allowed for the verification of the OIDC token.
+
+https://github.com/cs3org/reva/pull/3839

changelog/unreleased/ocm-propfind.md

@@ -0,0 +1,4 @@
+Bugfix: Fix propfind URL for OCM shares
+
+https://github.com/cs3org/reva/pull/3813
+https://github.com/cs3org/reva/issues/3810

changelog/unreleased/ocm-share-create-sm.md

@@ -0,0 +1,4 @@
+Enhancement: Create OCM share from sciencemesh service
+
+https://github.com/cs3org/reva/pull/3695
+https://github.com/pondersource/sciencemesh-php/issues/166

changelog/unreleased/revamp-rest-used-group-drivers.md

@@ -0,0 +1,11 @@
+Enhancement: Revamp user/group drivers and fix user type
+for lightweight accounts
+
+* Fix the user type for lightweight accounts, using the
+source field to differentiate between a primary and lw account
+* Remove all the code with manual parsing of the json returned
+by the CERN provider
+* Introduce pagination for `GetMembers` method in the group driver
+* Reduced network transfer size by requesting only needed fields for `GetMembers` method
+
+https://github.com/cs3org/reva/pull/3821

changelog/unreleased/serverless-services.md

@@ -0,0 +1,7 @@
+Enhancement: Serverless Services
+
+New type of service (along with http and grpc)
+which does not have a listening server. Useful for
+the notifications service and others in the future.
+
+https://github.com/cs3org/reva/pull/3824

cmd/reva/ocm-share-update-received.go

@@ -24,6 +24,7 @@ import (
 
 	rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
 	ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1"
+	typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
 	"github.com/pkg/errors"
 	"google.golang.org/protobuf/types/known/fieldmaskpb"
 )
@@ -33,9 +34,11 @@ func ocmShareUpdateReceivedCommand() *command {
 	cmd.Description = func() string { return "update a received OCM share" }
 	cmd.Usage = func() string { return "Usage: ocm-share-update-received [-flags] <share_id>" }
 	state := cmd.String("state", "pending", "the state of the share (pending, accepted or rejected)")
+	path := cmd.String("path", "", "the destination path of the data transfer (ignored if this is not a transfer type share)")
 
 	cmd.ResetFlags = func() {
 		*state = "pending"
+		*path = ""
 	}
 
 	cmd.Action = func(w ...io.Writer) error {
@@ -75,9 +78,25 @@ func ocmShareUpdateReceivedCommand() *command {
 		}
 		shareRes.Share.State = shareState
 
+		// check if we are dealing with a transfer in case the destination path needs to be set
+		_, ok := getTransferProtocol(shareRes.Share)
+		var opaque *typesv1beta1.Opaque
+		if ok {
+			// transfer_destination_path is not part of TransferProtocol and is specified as an opaque field
+			opaque = &typesv1beta1.Opaque{
+				Map: map[string]*typesv1beta1.OpaqueEntry{
+					"transfer_destination_path": {
+						Decoder: "plain",
+						Value:   []byte(*path),
+					},
+				},
+			}
+		}
+
 		shareRequest := &ocm.UpdateReceivedOCMShareRequest{
 			Share:      shareRes.Share,
 			UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"state"}},
+			Opaque:     opaque,
 		}
 
 		updateRes, err := shareClient.UpdateReceivedOCMShare(ctx, shareRequest)
@@ -95,6 +114,15 @@ func ocmShareUpdateReceivedCommand() *command {
 	return cmd
 }
 
+func getTransferProtocol(share *ocm.ReceivedShare) (*ocm.TransferProtocol, bool) {
+	for _, p := range share.Protocols {
+		if d, ok := p.Term.(*ocm.Protocol_TransferOptions); ok {
+			return d.TransferOptions, true
+		}
+	}
+	return nil, false
+}
+
 func getOCMShareState(state string) ocm.ShareState {
 	switch state {
 	case "pending":

cmd/revad/internal/grace/grace.go

@@ -41,6 +41,7 @@ type Watcher struct {
 	ppid      int
 	lns       map[string]net.Listener
 	ss        map[string]Server
+	SL        Serverless
 	pidFile   string
 	childPIDs []int
 }
@@ -254,6 +255,12 @@ type Server interface {
 	Address() string
 }
 
+// Serverless is the interface that the serverless server implements.
+type Serverless interface {
+	Stop() error
+	GracefulStop() error
+}
+
 // TrapSignals captures the OS signal.
 func (w *Watcher) TrapSignals() {
 	signalCh := make(chan os.Signal, 1024)
@@ -293,6 +300,11 @@ func (w *Watcher) TrapSignals() {
 							}
 							w.log.Info().Msgf("fd to %s:%s abruptly closed", s.Network(), s.Address())
 						}
+						err := w.SL.Stop()
+						if err != nil {
+							w.log.Error().Err(err).Msg("error stopping serverless server")
+						}
+						w.log.Info().Msg("serverless services abruptly closed")
 						w.Exit(1)
 					}
 				}
@@ -306,6 +318,14 @@ func (w *Watcher) TrapSignals() {
 					w.Exit(1)
 				}
 			}
+			if w.SL != nil {
+				err := w.SL.GracefulStop()
+				if err != nil {
+					w.log.Error().Err(err).Msg("error stopping server")
+					w.log.Info().Msg("exit with error code 1")
+					w.Exit(1)
+				}
+			}
 			w.log.Info().Msg("exit with error code 0")
 			w.Exit(0)
 		case syscall.SIGINT, syscall.SIGTERM:
@@ -317,6 +337,12 @@ func (w *Watcher) TrapSignals() {
 					w.log.Error().Err(err).Msg("error stopping server")
 				}
 			}
+			err := w.SL.Stop()
+			if err != nil {
+				w.log.Error().Err(err).Msg("error stopping serverless server")
+			}
+			w.log.Info().Msg("serverless services abruptly closed")
+
 			w.Exit(0)
 		}
 	}

cmd/revad/runtime/loader.go

@@ -27,6 +27,7 @@ import (
 	_ "github.com/cs3org/reva/internal/http/interceptors/auth/tokenwriter/loader"
 	_ "github.com/cs3org/reva/internal/http/interceptors/loader"
 	_ "github.com/cs3org/reva/internal/http/services/loader"
+	_ "github.com/cs3org/reva/internal/serverless/services/loader"
 	_ "github.com/cs3org/reva/pkg/app/provider/loader"
 	_ "github.com/cs3org/reva/pkg/app/registry/loader"
 	_ "github.com/cs3org/reva/pkg/appauth/manager/loader"

cmd/revad/runtime/runtime.go

@@ -33,6 +33,7 @@ import (
 	"github.com/cs3org/reva/pkg/registry/memory"
 	"github.com/cs3org/reva/pkg/rgrpc"
 	"github.com/cs3org/reva/pkg/rhttp"
+	"github.com/cs3org/reva/pkg/rserverless"
 	"github.com/cs3org/reva/pkg/sharedconf"
 	rtrace "github.com/cs3org/reva/pkg/trace"
 	"github.com/cs3org/reva/pkg/utils"
@@ -93,13 +94,23 @@ func run(mainConf map[string]interface{}, coreConf *coreConf, logger *zerolog.Lo
 	initCPUCount(coreConf, logger)
 
 	servers := initServers(mainConf, logger)
+	serverless := initServerless(mainConf, logger)
+
+	if len(servers) == 0 && serverless == nil {
+		logger.Info().Msg("nothing to do, no grpc/http/serverless enabled_services declared in config")
+		os.Exit(1)
+	}
+
 	watcher, err := initWatcher(logger, filename)
 	if err != nil {
 		log.Panic(err)
 	}
 	listeners := initListeners(watcher, servers, logger)
+	if serverless != nil {
+		watcher.SL = serverless
+	}
 
-	start(mainConf, servers, listeners, logger, watcher)
+	start(mainConf, servers, serverless, listeners, logger, watcher)
 }
 
 func initListeners(watcher *grace.Watcher, servers map[string]grace.Server, log *zerolog.Logger) map[string]net.Listener {
@@ -141,13 +152,22 @@ func initServers(mainConf map[string]interface{}, log *zerolog.Logger) map[strin
 		servers["grpc"] = s
 	}
 
-	if len(servers) == 0 {
-		log.Info().Msg("nothing to do, no grpc/http enabled_services declared in config")
-		os.Exit(1)
-	}
 	return servers
 }
 
+func initServerless(mainConf map[string]interface{}, log *zerolog.Logger) *rserverless.Serverless {
+	if isEnabledServerless(mainConf) {
+		serverless, err := getServerless(mainConf["serverless"], log)
+		if err != nil {
+			log.Error().Err(err).Msg("error")
+			os.Exit(1)
+		}
+		return serverless
+	}
+
+	return nil
+}
+
 func initTracing(conf *coreConf) {
 	rtrace.SetTraceProvider(conf.TracingCollector, conf.TracingEndpoint, conf.TracingServiceName)
 }
@@ -184,7 +204,7 @@ func handlePIDFlag(l *zerolog.Logger, pidFile string) (*grace.Watcher, error) {
 	return w, nil
 }
 
-func start(mainConf map[string]interface{}, servers map[string]grace.Server, listeners map[string]net.Listener, log *zerolog.Logger, watcher *grace.Watcher) {
+func start(mainConf map[string]interface{}, servers map[string]grace.Server, serverless *rserverless.Serverless, listeners map[string]net.Listener, log *zerolog.Logger, watcher *grace.Watcher) {
 	if isEnabledHTTP(mainConf) {
 		go func() {
 			if err := servers["http"].(*rhttp.Server).Start(listeners["http"]); err != nil {
@@ -201,6 +221,13 @@ func start(mainConf map[string]interface{}, servers map[string]grace.Server, lis
 			}
 		}()
 	}
+	if isEnabledServerless(mainConf) {
+		if err := serverless.Start(); err != nil {
+			log.Error().Err(err).Msg("error starting serverless services")
+			watcher.Exit(1)
+		}
+	}
+
 	watcher.TrapSignals()
 }
 
@@ -264,6 +291,11 @@ func getHTTPServer(conf interface{}, l *zerolog.Logger) (*rhttp.Server, error) {
 	return s, nil
 }
 
+func getServerless(conf interface{}, l *zerolog.Logger) (*rserverless.Serverless, error) {
+	sub := l.With().Str("pkg", "rserverless").Logger()
+	return rserverless.New(conf, sub)
+}
+
 //	adjustCPU parses string cpu and sets GOMAXPROCS
 //
 // according to its value. It accepts either
@@ -365,6 +397,10 @@ func isEnabledGRPC(conf map[string]interface{}) bool {
 	return isEnabled("grpc", conf)
 }
 
+func isEnabledServerless(conf map[string]interface{}) bool {
+	return isEnabled("serverless", conf)
+}
+
 func isEnabled(key string, conf map[string]interface{}) bool {
 	if a, ok := conf[key]; ok {
 		if b, ok := a.(map[string]interface{}); ok {

docs/content/en/docs/tutorials/share-tutorial.md

@@ -158,7 +158,7 @@ In this case, the share can be accessed using the WebDAV protocol (multiple acce
 For example:
 ```
 # curl -X PROPFIND http://localhost:19001/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D
-<?xml version="1.0" encoding="utf-8"?><d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns"><d:response><d:href>/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D/home/my-folder/</d:href><d:propstat><d:prop><oc:id>123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder</oc:id><oc:fileid>123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder</oc:fileid><d:getetag>&#34;e35fa97883e0481aabf235abb8eb6b1f&#34;</d:getetag><oc:permissions>SDNVCK</oc:permissions><d:resourcetype><d:collection/></d:resourcetype><oc:size>25</oc:size><d:getlastmodified>Tue, 11 Apr 2023 09:56:29 GMT</d:getlastmodified><oc:favorite>0</oc:favorite></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response><d:response><d:href>/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D/home/my-folder/example.txt</d:href><d:propstat><d:prop><oc:id>123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder%2Fexample.txt</oc:id><oc:fileid>123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder%2Fexample.txt</oc:fileid><d:getetag>&#34;bf73fa7d3ebf18b3cff6d64ed25a7de0&#34;</d:getetag><oc:permissions>SDNVW</oc:permissions><d:resourcetype></d:resourcetype><d:getcontentlength>33</d:getcontentlength><d:getcontenttype>text/plain</d:getcontenttype><d:getlastmodified>Tue, 11 Apr 2023 09:56:29 GMT</d:getlastmodified><oc:favorite>0</oc:favorite></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>
+<?xml version="1.0" encoding="utf-8"?><d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns"><d:response><d:href>/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D/</d:href><d:propstat><d:prop><oc:id>123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder</oc:id><oc:fileid>123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder</oc:fileid><d:getetag>&#34;e35fa97883e0481aabf235abb8eb6b1f&#34;</d:getetag><oc:permissions>SDNVCK</oc:permissions><d:resourcetype><d:collection/></d:resourcetype><oc:size>25</oc:size><d:getlastmodified>Tue, 11 Apr 2023 09:56:29 GMT</d:getlastmodified><oc:favorite>0</oc:favorite></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response><d:response><d:href>/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D/example.txt</d:href><d:propstat><d:prop><oc:id>123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder%2Fexample.txt</oc:id><oc:fileid>123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder%2Fexample.txt</oc:fileid><d:getetag>&#34;bf73fa7d3ebf18b3cff6d64ed25a7de0&#34;</d:getetag><oc:permissions>SDNVW</oc:permissions><d:resourcetype></d:resourcetype><d:getcontentlength>33</d:getcontentlength><d:getcontenttype>text/plain</d:getcontenttype><d:getlastmodified>Tue, 11 Apr 2023 09:56:29 GMT</d:getlastmodified><oc:favorite>0</oc:favorite></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>
 ```
 
 In particular, reva allows an user to navigate the received shares in a more user-friendly way, exposing the shares under the `/sciencemesh` mount point. The format to access a received share is `/sciencemesh/<share-id>[/<relative-path>]`.