From 2613cf8436429b88ccd8559117635b0c214d7ba0 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Thu, 20 Apr 2023 17:24:46 +0200 Subject: [PATCH 01/20] apps: fixed viewMode resolution by making permissions override user's choices (#3805) --- changelog/unreleased/apps-viewmode.md | 6 ++++++ internal/http/services/appprovider/appprovider.go | 14 +++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/apps-viewmode.md diff --git a/changelog/unreleased/apps-viewmode.md b/changelog/unreleased/apps-viewmode.md new file mode 100644 index 0000000000..634c08d3f7 --- /dev/null +++ b/changelog/unreleased/apps-viewmode.md @@ -0,0 +1,6 @@ +Bugfix: Apps: fixed viewMode resolution + +Currently, the viewMode passed on /app/open is taken without validating +the actual user's permissions. This PR fixes this. + +https://github.com/cs3org/reva/pull/3805 diff --git a/internal/http/services/appprovider/appprovider.go b/internal/http/services/appprovider/appprovider.go index c4366dc148..c57d9fbc78 100644 --- a/internal/http/services/appprovider/appprovider.go +++ b/internal/http/services/appprovider/appprovider.go @@ -449,19 +449,23 @@ func filterAppsByUserAgent(mimeTypes []*appregistry.MimeTypeInfo, userAgent stri } func resolveViewMode(res *provider.ResourceInfo, vm string) gateway.OpenInAppRequest_ViewMode { + var viewMode gateway.OpenInAppRequest_ViewMode if vm != "" { - return utils.GetViewMode(vm) + viewMode = utils.GetViewMode(vm) + } else { + viewMode = gateway.OpenInAppRequest_VIEW_MODE_READ_WRITE } - - var viewMode gateway.OpenInAppRequest_ViewMode canEdit := res.PermissionSet.InitiateFileUpload canView := res.PermissionSet.InitiateFileDownload switch { case canEdit && canView: - viewMode = gateway.OpenInAppRequest_VIEW_MODE_READ_WRITE + // ok case canView: - viewMode = gateway.OpenInAppRequest_VIEW_MODE_READ_ONLY + if viewMode == gateway.OpenInAppRequest_VIEW_MODE_READ_WRITE || viewMode == gateway.OpenInAppRequest_VIEW_MODE_PREVIEW { + // downgrade to the maximum permitted viewmode + viewMode = gateway.OpenInAppRequest_VIEW_MODE_READ_ONLY + } default: // no permissions, will return access denied viewMode = gateway.OpenInAppRequest_VIEW_MODE_INVALID From bf56c237f3c9773def5f6048e91ee17c3af065ff Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Tue, 25 Apr 2023 11:40:04 +0200 Subject: [PATCH 02/20] Fix propfind URL for OCM shares (#3813) * fix propfind result for ocm share * fix doc * add changelog --- changelog/unreleased/ocm-propfind.md | 4 + .../en/docs/tutorials/share-tutorial.md | 2 +- pkg/ocm/storage/outcoming/ocm.go | 89 +++++++++---------- 3 files changed, 49 insertions(+), 46 deletions(-) create mode 100644 changelog/unreleased/ocm-propfind.md diff --git a/changelog/unreleased/ocm-propfind.md b/changelog/unreleased/ocm-propfind.md new file mode 100644 index 0000000000..f97b0bb093 --- /dev/null +++ b/changelog/unreleased/ocm-propfind.md @@ -0,0 +1,4 @@ +Bugfix: Fix propfind URL for OCM shares + +https://github.com/cs3org/reva/pull/3813 +https://github.com/cs3org/reva/issues/3810 diff --git a/docs/content/en/docs/tutorials/share-tutorial.md b/docs/content/en/docs/tutorials/share-tutorial.md index 9ded80828c..153c68a08e 100644 --- a/docs/content/en/docs/tutorials/share-tutorial.md +++ b/docs/content/en/docs/tutorials/share-tutorial.md @@ -158,7 +158,7 @@ In this case, the share can be accessed using the WebDAV protocol (multiple acce For example: ``` # curl -X PROPFIND http://localhost:19001/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D -/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D/home/my-folder/123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder"e35fa97883e0481aabf235abb8eb6b1f"SDNVCK25Tue, 11 Apr 2023 09:56:29 GMT0HTTP/1.1 200 OK/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D/home/my-folder/example.txt123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder%2Fexample.txt123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder%2Fexample.txt"bf73fa7d3ebf18b3cff6d64ed25a7de0"SDNVW33text/plainTue, 11 Apr 2023 09:56:29 GMT0HTTP/1.1 200 OK +/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D/123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder"e35fa97883e0481aabf235abb8eb6b1f"SDNVCK25Tue, 11 Apr 2023 09:56:29 GMT0HTTP/1.1 200 OK/remote.php/dav/ocm/eSWNjTWjorFmZEGQNZVyrU3TyxdWEr1D/example.txt123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder%2Fexample.txt123e4567-e89b-12d3-a456-426655440000!fileid-einstein%2Fmy-folder%2Fexample.txt"bf73fa7d3ebf18b3cff6d64ed25a7de0"SDNVW33text/plainTue, 11 Apr 2023 09:56:29 GMT0HTTP/1.1 200 OK ``` In particular, reva allows an user to navigate the received shares in a more user-friendly way, exposing the shares under the `/sciencemesh` mount point. The format to access a received share is `/sciencemesh/[/]`. diff --git a/pkg/ocm/storage/outcoming/ocm.go b/pkg/ocm/storage/outcoming/ocm.go index 672dc89ec1..5c179d4e16 100644 --- a/pkg/ocm/storage/outcoming/ocm.go +++ b/pkg/ocm/storage/outcoming/ocm.go @@ -176,8 +176,8 @@ func (d *driver) CreateDir(ctx context.Context, ref *provider.Reference) error { return err } - return d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, ref *provider.Reference) error { - res, err := d.gateway.CreateContainer(userCtx, &provider.CreateContainerRequest{Ref: ref}) + return d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, ref *provider.Reference) error { + res, err := d.gateway.CreateContainer(ctx, &provider.CreateContainerRequest{Ref: ref}) switch { case err != nil: return err @@ -195,8 +195,8 @@ func (d *driver) TouchFile(ctx context.Context, ref *provider.Reference) error { return err } - return d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, ref *provider.Reference) error { - res, err := d.gateway.TouchFile(userCtx, &provider.TouchFileRequest{Ref: ref}) + return d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, ref *provider.Reference) error { + res, err := d.gateway.TouchFile(ctx, &provider.TouchFileRequest{Ref: ref}) switch { case err != nil: return err @@ -214,8 +214,8 @@ func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error { return err } - return d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, ref *provider.Reference) error { - res, err := d.gateway.Delete(userCtx, &provider.DeleteRequest{Ref: ref}) + return d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, ref *provider.Reference) error { + res, err := d.gateway.Delete(ctx, &provider.DeleteRequest{Ref: ref}) switch { case err != nil: return err @@ -263,12 +263,12 @@ func (d *driver) opFromUser(ctx context.Context, userID *userv1beta1.UserId, f f } func (d *driver) unwrappedOpFromShareCreator(ctx context.Context, share *ocmv1beta1.Share, rel string, f func(ctx context.Context, ref *provider.Reference) error) error { - return d.opFromUser(ctx, share.Creator, func(userCtx context.Context) error { - newRef, err := d.translateOCMShareResourceToCS3Ref(userCtx, share.ResourceId, rel) + return d.opFromUser(ctx, share.Creator, func(ctx context.Context) error { + newRef, err := d.translateOCMShareResourceToCS3Ref(ctx, share.ResourceId, rel) if err != nil { return err } - return f(userCtx, newRef) + return f(ctx, newRef) }) } @@ -279,17 +279,16 @@ func (d *driver) GetMD(ctx context.Context, ref *provider.Reference, _ []string) } var info *provider.ResourceInfo - if err := d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, newRef *provider.Reference) error { - info, err = d.stat(userCtx, newRef) - return err + if err := d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, newRef *provider.Reference) error { + info, err = d.stat(ctx, newRef) + if err != nil { + return err + } + return d.augmentResourceInfo(ctx, info, share) }); err != nil { return nil, err } - if err := d.augmentResourceInfo(ctx, info, share); err != nil { - return nil, err - } - return info, nil } @@ -338,8 +337,8 @@ func (d *driver) ListFolder(ctx context.Context, ref *provider.Reference, _ []st } var infos []*provider.ResourceInfo - if err := d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, newRef *provider.Reference) error { - lstRes, err := d.gateway.ListContainer(userCtx, &provider.ListContainerRequest{Ref: newRef}) + if err := d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, newRef *provider.Reference) error { + lstRes, err := d.gateway.ListContainer(ctx, &provider.ListContainerRequest{Ref: newRef}) switch { case err != nil: return err @@ -349,21 +348,21 @@ func (d *driver) ListFolder(ctx context.Context, ref *provider.Reference, _ []st return errtypes.InternalError(lstRes.Status.Message) } infos = lstRes.Infos + + shareInfo, err := d.stat(ctx, &provider.Reference{ResourceId: share.ResourceId}) + if err != nil { + return err + } + + perms := getPermissionsFromShare(share) + for _, info := range infos { + fixResourceInfo(info, shareInfo, share, perms) + } return nil }); err != nil { return nil, err } - shareInfo, err := d.stat(ctx, &provider.Reference{ResourceId: share.ResourceId}) - if err != nil { - return nil, err - } - - perms := getPermissionsFromShare(share) - for _, info := range infos { - fixResourceInfo(info, shareInfo, share, perms) - } - return infos, nil } @@ -403,8 +402,8 @@ func (d *driver) Upload(ctx context.Context, ref *provider.Reference, content io return err } - return d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, newRef *provider.Reference) error { - initRes, err := d.gateway.InitiateFileUpload(userCtx, &provider.InitiateFileUploadRequest{Ref: newRef}) + return d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, newRef *provider.Reference) error { + initRes, err := d.gateway.InitiateFileUpload(ctx, &provider.InitiateFileUploadRequest{Ref: newRef}) switch { case err != nil: return err @@ -417,7 +416,7 @@ func (d *driver) Upload(ctx context.Context, ref *provider.Reference, content io return errtypes.InternalError("simple upload not supported") } - httpReq, err := rhttp.NewRequest(userCtx, http.MethodPut, endpoint, content) + httpReq, err := rhttp.NewRequest(ctx, http.MethodPut, endpoint, content) if err != nil { return errors.Wrap(err, "error creating new request") } @@ -456,8 +455,8 @@ func (d *driver) Download(ctx context.Context, ref *provider.Reference) (io.Read } var r io.ReadCloser - if err := d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, newRef *provider.Reference) error { - initRes, err := d.gateway.InitiateFileDownload(userCtx, &provider.InitiateFileDownloadRequest{Ref: newRef}) + if err := d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, newRef *provider.Reference) error { + initRes, err := d.gateway.InitiateFileDownload(ctx, &provider.InitiateFileDownloadRequest{Ref: newRef}) switch { case err != nil: return err @@ -472,7 +471,7 @@ func (d *driver) Download(ctx context.Context, ref *provider.Reference) (io.Read return errtypes.InternalError("simple download not supported") } - httpReq, err := rhttp.NewRequest(userCtx, http.MethodGet, endpoint, nil) + httpReq, err := rhttp.NewRequest(ctx, http.MethodGet, endpoint, nil) if err != nil { return err } @@ -509,7 +508,7 @@ func (d *driver) SetLock(ctx context.Context, ref *provider.Reference, lock *pro return err } - return d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, newRef *provider.Reference) error { + return d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, newRef *provider.Reference) error { lockRes, err := d.gateway.SetLock(ctx, &provider.SetLockRequest{ Ref: newRef, Lock: lock, @@ -535,8 +534,8 @@ func (d *driver) GetLock(ctx context.Context, ref *provider.Reference) (*provide } var lock *provider.Lock - if err := d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, newRef *provider.Reference) error { - lockRes, err := d.gateway.GetLock(userCtx, &provider.GetLockRequest{Ref: newRef}) + if err := d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, newRef *provider.Reference) error { + lockRes, err := d.gateway.GetLock(ctx, &provider.GetLockRequest{Ref: newRef}) switch { case err != nil: return err @@ -560,8 +559,8 @@ func (d *driver) RefreshLock(ctx context.Context, ref *provider.Reference, lock return err } - return d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, newRef *provider.Reference) error { - lockRes, err := d.gateway.RefreshLock(userCtx, &provider.RefreshLockRequest{ + return d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, newRef *provider.Reference) error { + lockRes, err := d.gateway.RefreshLock(ctx, &provider.RefreshLockRequest{ Ref: newRef, ExistingLockId: existingLockID, Lock: lock, @@ -586,8 +585,8 @@ func (d *driver) Unlock(ctx context.Context, ref *provider.Reference, lock *prov return err } - return d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, newRef *provider.Reference) error { - lockRes, err := d.gateway.Unlock(userCtx, &provider.UnlockRequest{ + return d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, newRef *provider.Reference) error { + lockRes, err := d.gateway.Unlock(ctx, &provider.UnlockRequest{ Ref: newRef, Lock: lock, }) @@ -611,8 +610,8 @@ func (d *driver) SetArbitraryMetadata(ctx context.Context, ref *provider.Referen return err } - return d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, newRef *provider.Reference) error { - res, err := d.gateway.SetArbitraryMetadata(userCtx, &provider.SetArbitraryMetadataRequest{ + return d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, newRef *provider.Reference) error { + res, err := d.gateway.SetArbitraryMetadata(ctx, &provider.SetArbitraryMetadataRequest{ Ref: newRef, ArbitraryMetadata: md, }) @@ -634,8 +633,8 @@ func (d *driver) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Refer return err } - return d.unwrappedOpFromShareCreator(ctx, share, rel, func(userCtx context.Context, newRef *provider.Reference) error { - res, err := d.gateway.UnsetArbitraryMetadata(userCtx, &provider.UnsetArbitraryMetadataRequest{ + return d.unwrappedOpFromShareCreator(ctx, share, rel, func(ctx context.Context, newRef *provider.Reference) error { + res, err := d.gateway.UnsetArbitraryMetadata(ctx, &provider.UnsetArbitraryMetadataRequest{ Ref: newRef, ArbitraryMetadataKeys: keys, }) From 21129fae8aa98ce7d81257d910f4c4f723b826d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 23 Mar 2023 11:25:48 +0100 Subject: [PATCH 03/20] Expose service states --- pkg/mentix/connectors/gocdb.go | 20 +++++++++++++------- pkg/mentix/connectors/gocdb/types.go | 14 ++++++++------ pkg/mentix/meshdata/service.go | 14 ++++++++------ 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index bee88d25a9..0cfca00f36 100644 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -230,7 +230,7 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site Name: endpoint.Name, RawURL: endpoint.URL, URL: getServiceURLString(service, endpoint, host), - IsMonitored: strings.EqualFold(endpoint.IsMonitored, "Y"), + IsMonitored: connector.convertStringToBool(endpoint.IsMonitored), Properties: connector.extensionsToMap(&endpoint.Extensions), }) } @@ -238,12 +238,14 @@ func (connector *GOCDBConnector) queryServices(meshData *meshdata.MeshData, site // Add the service to the site site.Services = append(site.Services, &meshdata.Service{ ServiceEndpoint: &meshdata.ServiceEndpoint{ - Type: connector.findServiceType(meshData, service.Type), - Name: service.Type, - RawURL: service.URL, - URL: getServiceURLString(service, nil, host), - IsMonitored: strings.EqualFold(service.IsMonitored, "Y"), - Properties: connector.extensionsToMap(&service.Extensions), + Type: connector.findServiceType(meshData, service.Type), + Name: service.Type, + RawURL: service.URL, + URL: getServiceURLString(service, nil, host), + IsInProduction: connector.convertStringToBool(service.IsInProduction), + IsBeta: connector.convertStringToBool(service.IsBeta), + IsMonitored: connector.convertStringToBool(service.IsMonitored), + Properties: connector.extensionsToMap(&service.Extensions), }, Host: host, AdditionalEndpoints: endpoints, @@ -332,6 +334,10 @@ func (connector *GOCDBConnector) getServiceURL(service *gocdb.Service, endpoint return svcURL, nil } +func (connector *GOCDBConnector) convertStringToBool(s string) bool { + return strings.EqualFold(s, "Y") +} + // GetID returns the ID of the connector. func (connector *GOCDBConnector) GetID() string { return config.ConnectorIDGOCDB diff --git a/pkg/mentix/connectors/gocdb/types.go b/pkg/mentix/connectors/gocdb/types.go index c5b37c7528..291272c1dc 100755 --- a/pkg/mentix/connectors/gocdb/types.go +++ b/pkg/mentix/connectors/gocdb/types.go @@ -89,12 +89,14 @@ type ServiceEndpoints struct { // Service represents a service in GOCDB. type Service struct { - Host string `xml:"HOSTNAME"` - Type string `xml:"SERVICE_TYPE"` - IsMonitored string `xml:"NODE_MONITORED"` - URL string `xml:"URL"` - Endpoints ServiceEndpoints `xml:"ENDPOINTS"` - Extensions Extensions `xml:"EXTENSIONS"` + Host string `xml:"HOSTNAME"` + Type string `xml:"SERVICE_TYPE"` + IsInProduction string `xml:"IN_PRODUCTION"` + IsBeta string `xml:"BETA"` + IsMonitored string `xml:"NODE_MONITORED"` + URL string `xml:"URL"` + Endpoints ServiceEndpoints `xml:"ENDPOINTS"` + Extensions Extensions `xml:"EXTENSIONS"` } // Services is a list of Service objects. diff --git a/pkg/mentix/meshdata/service.go b/pkg/mentix/meshdata/service.go index 3310f6dad2..29471bde80 100644 --- a/pkg/mentix/meshdata/service.go +++ b/pkg/mentix/meshdata/service.go @@ -92,12 +92,14 @@ func (serviceType *ServiceType) Verify() error { // ServiceEndpoint represents a service endpoint managed by Mentix. type ServiceEndpoint struct { - Type *ServiceType - Name string - RawURL string - URL string - IsMonitored bool - Properties map[string]string + Type *ServiceType + Name string + RawURL string + URL string + IsInProduction bool + IsBeta bool + IsMonitored bool + Properties map[string]string } // InferMissingData infers missing data from other data where possible. From a8612dd87caf10ca3e346ae78174e08f4e8a609d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 23 Mar 2023 11:52:29 +0100 Subject: [PATCH 04/20] Expose new flags through cs3 endpoint --- pkg/mentix/connectors/gocdb.go | 32 ++++++++++--------- pkg/mentix/connectors/gocdb/types.go | 24 +++++++------- .../exchangers/exporters/cs3api/query.go | 21 ++++++++++-- pkg/mentix/meshdata/properties.go | 9 ++++++ pkg/mentix/meshdata/site.go | 28 ++++++++-------- 5 files changed, 73 insertions(+), 41 deletions(-) diff --git a/pkg/mentix/connectors/gocdb.go b/pkg/mentix/connectors/gocdb.go index 0cfca00f36..1d1ecf0f05 100644 --- a/pkg/mentix/connectors/gocdb.go +++ b/pkg/mentix/connectors/gocdb.go @@ -172,21 +172,23 @@ func (connector *GOCDBConnector) querySites(meshData *meshdata.MeshData, op *mes organization := meshdata.GetPropertyValue(properties, meshdata.PropertyOrganization, site.OfficialName) meshsite := &meshdata.Site{ - ID: siteID, - Name: site.ShortName, - FullName: site.OfficialName, - Organization: organization, - Domain: site.Domain, - Homepage: site.Homepage, - Email: site.Email, - Description: site.Description, - Country: site.Country, - CountryCode: site.CountryCode, - Longitude: site.Longitude, - Latitude: site.Latitude, - Services: nil, - Properties: properties, - Downtimes: meshdata.Downtimes{}, + ID: siteID, + Name: site.ShortName, + FullName: site.OfficialName, + Organization: organization, + Domain: site.Domain, + Infrastructure: site.Infrastructure, + Certification: site.Certification, + Homepage: site.Homepage, + Email: site.Email, + Description: site.Description, + Country: site.Country, + CountryCode: site.CountryCode, + Longitude: site.Longitude, + Latitude: site.Latitude, + Services: nil, + Properties: properties, + Downtimes: meshdata.Downtimes{}, } op.Sites = append(op.Sites, meshsite) } diff --git a/pkg/mentix/connectors/gocdb/types.go b/pkg/mentix/connectors/gocdb/types.go index 291272c1dc..73bb3a6e8c 100755 --- a/pkg/mentix/connectors/gocdb/types.go +++ b/pkg/mentix/connectors/gocdb/types.go @@ -55,17 +55,19 @@ type NGIs struct { // Site represents a site in GOCDB. type Site struct { - ShortName string `xml:"SHORT_NAME"` - OfficialName string `xml:"OFFICIAL_NAME"` - Description string `xml:"SITE_DESCRIPTION"` - Homepage string `xml:"HOME_URL"` - Email string `xml:"CONTACT_EMAIL"` - Domain string `xml:"DOMAIN>DOMAIN_NAME"` - Country string `xml:"COUNTRY"` - CountryCode string `xml:"COUNTRY_CODE"` - Latitude float32 `xml:"LATITUDE"` - Longitude float32 `xml:"LONGITUDE"` - Extensions Extensions `xml:"EXTENSIONS"` + ShortName string `xml:"SHORT_NAME"` + OfficialName string `xml:"OFFICIAL_NAME"` + Description string `xml:"SITE_DESCRIPTION"` + Homepage string `xml:"HOME_URL"` + Email string `xml:"CONTACT_EMAIL"` + Domain string `xml:"DOMAIN>DOMAIN_NAME"` + Infrastructure string `xml:"PRODUCTION_INFRASTRUCTURE"` + Certification string `xml:"CERTIFICATION_STATUS"` + Country string `xml:"COUNTRY"` + CountryCode string `xml:"COUNTRY_CODE"` + Latitude float32 `xml:"LATITUDE"` + Longitude float32 `xml:"LONGITUDE"` + Extensions Extensions `xml:"EXTENSIONS"` } // Sites is a list of Site objects. diff --git a/pkg/mentix/exchangers/exporters/cs3api/query.go b/pkg/mentix/exchangers/exporters/cs3api/query.go index e8652aa7d0..9736a58a95 100644 --- a/pkg/mentix/exchangers/exporters/cs3api/query.go +++ b/pkg/mentix/exchangers/exporters/cs3api/query.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "strings" ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" @@ -95,7 +96,11 @@ func convertMeshDataToOCMData(meshData *meshdata.MeshData, elevatedServiceTypes Services: services, Properties: site.Properties, } - provider.Properties[strings.ToUpper(meshdata.PropertyOperator)] = op.ID // Propagate the operator ID as a property + // Propagate the operator ID as a property + setPropertyValue(provider.Properties, meshdata.PropertyOperator, op.ID) + // Expose additional site details as properties + setPropertyValue(provider.Properties, meshdata.PropertyInfrastructure, site.Infrastructure) + setPropertyValue(provider.Properties, meshdata.PropertyCertification, site.Certification) providers = append(providers, provider) } } @@ -103,6 +108,14 @@ func convertMeshDataToOCMData(meshData *meshdata.MeshData, elevatedServiceTypes } func convertServiceEndpointToOCMData(endpoint *meshdata.ServiceEndpoint, log *zerolog.Logger) *ocmprovider.ServiceEndpoint { + properties := make(map[string]string, 10) + for k, v := range endpoint.Properties { + properties[k] = v + } + // Expose additional service details as properties + setPropertyValue(properties, meshdata.PropertyIsInProduction, strconv.FormatBool(endpoint.IsInProduction)) + setPropertyValue(properties, meshdata.PropertyIsBeta, strconv.FormatBool(endpoint.IsBeta)) + return &ocmprovider.ServiceEndpoint{ Type: &ocmprovider.ServiceType{ Name: endpoint.Type.Name, @@ -111,6 +124,10 @@ func convertServiceEndpointToOCMData(endpoint *meshdata.ServiceEndpoint, log *ze Name: endpoint.Name, Path: normalizeURLPath(endpoint.URL, log), IsMonitored: endpoint.IsMonitored, - Properties: endpoint.Properties, + Properties: properties, } } + +func setPropertyValue(properties map[string]string, key string, value string) { + properties[strings.ToUpper(key)] = value +} diff --git a/pkg/mentix/meshdata/properties.go b/pkg/mentix/meshdata/properties.go index 3e04bd193f..a2cc6e282b 100644 --- a/pkg/mentix/meshdata/properties.go +++ b/pkg/mentix/meshdata/properties.go @@ -30,6 +30,15 @@ const ( // PropertyAPIVersion identifies the API version property. PropertyAPIVersion = "api_version" + + // PropertyInfrastructure identifies the infrastructure type of a site. + PropertyInfrastructure = "infrastructure" + // PropertyCertification identifies the certification status of a site. + PropertyCertification = "certification" + // PropertyIsInProduction identifies if a service is in production. + PropertyIsInProduction = "in_production" + // PropertyIsBeta identifies if a service is in beta. + PropertyIsBeta = "beta" ) // GetPropertyValue performs a case-insensitive search for the given property. diff --git a/pkg/mentix/meshdata/site.go b/pkg/mentix/meshdata/site.go index 1a7459a232..7840e2b17d 100644 --- a/pkg/mentix/meshdata/site.go +++ b/pkg/mentix/meshdata/site.go @@ -28,19 +28,21 @@ import ( // Site represents a single site managed by Mentix. type Site struct { - ID string - Name string - FullName string - Organization string - Domain string - Homepage string - Email string - Description string - Country string - CountryCode string - Location string - Latitude float32 - Longitude float32 + ID string + Name string + FullName string + Organization string + Domain string + Infrastructure string + Certification string + Homepage string + Email string + Description string + Country string + CountryCode string + Location string + Latitude float32 + Longitude float32 Services []*Service Properties map[string]string From bb09c12925d91d01056f4be111f99a94ebe031bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Fri, 24 Mar 2023 14:16:24 +0100 Subject: [PATCH 05/20] Add changelog --- changelog/unreleased/mentix-prodflags.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/mentix-prodflags.md diff --git a/changelog/unreleased/mentix-prodflags.md b/changelog/unreleased/mentix-prodflags.md new file mode 100644 index 0000000000..89d3c898be --- /dev/null +++ b/changelog/unreleased/mentix-prodflags.md @@ -0,0 +1,5 @@ +Enhancement: New metadata flags + +Several new flags, like site infrastructure and service status, are now gathered and exposed by Mentix. + +https://github.com/cs3org/reva/pull/3750 From f2fca5a5028db575445d1f7bf801ab19ed0b96f7 Mon Sep 17 00:00:00 2001 From: Sagar Gurung <46086950+SagarGi@users.noreply.github.com> Date: Wed, 26 Apr 2023 14:08:21 +0545 Subject: [PATCH 06/20] [tests-only][full-ci]Bump latest-ocis commit to reva master (#3776) * Bump latest commit of ocis on reva * update expexted to failure lines * Add skip on master on compose file * Take latest ocis * Took ocis commit 25 apirl --- .drone.env | 2 +- .drone.star | 4 ++-- tests/acceptance/expected-failures-on-OCIS-storage.md | 10 +++++----- tests/acceptance/expected-failures-on-S3NG-storage.md | 10 +++++----- tests/docker/docker-compose.yml | 4 ++-- tests/ocis | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.drone.env b/.drone.env index 3a568199df..e774b092a9 100644 --- a/.drone.env +++ b/.drone.env @@ -1,4 +1,4 @@ # The test runner source for API tests -APITESTS_COMMITID=76cc388e4546d4e588515c6c7d624829a674163a +APITESTS_COMMITID=4076bc71e63b4dd4d7931e1ad085ee214f137e38 APITESTS_BRANCH=master APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/.drone.star b/.drone.star index c55ea1327e..e80cb94fb1 100644 --- a/.drone.star +++ b/.drone.star @@ -119,7 +119,7 @@ def ocisIntegrationTest(): "REVA_LDAP_HOSTNAME": "ldap", "TEST_REVA": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva", + "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster", "DIVIDE_INTO_NUM_PARTS": 6, "RUN_PART": 2, "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md", @@ -190,7 +190,7 @@ def s3ngIntegrationTests(): "REVA_LDAP_HOSTNAME": "ldap", "TEST_REVA": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", - "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva", + "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster", "DIVIDE_INTO_NUM_PARTS": parallelRuns, "RUN_PART": runPart, "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md", diff --git a/tests/acceptance/expected-failures-on-OCIS-storage.md b/tests/acceptance/expected-failures-on-OCIS-storage.md index 21aaff79d5..a38d5ffbcb 100644 --- a/tests/acceptance/expected-failures-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-on-OCIS-storage.md @@ -303,11 +303,11 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiWebdavMove2/moveFile.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L177) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) //todo -- [coreApiVersions/fileVersions.feature:291](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L291) +- [coreApiVersions/fileVersions.feature:276](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L276) #### [restoring an older version of a shared file deletes the share](https://github.com/owncloud/ocis/issues/765) - [coreApiShareManagementToShares/acceptShares.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L483) -- [coreApiVersions/fileVersions.feature:302](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L302) +- [coreApiVersions/fileVersions.feature:288](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L288) #### [Expiration date for shares is not implemented](https://github.com/owncloud/ocis/issues/1250) #### Expiration date of user shares @@ -397,14 +397,14 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiShareUpdateToShares/updateShare.feature:241](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L241) #### [user can access version metadata of a received share before accepting it](https://github.com/owncloud/ocis/issues/760) -- [coreApiVersions/fileVersions.feature:487](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L487) +- [coreApiVersions/fileVersions.feature:313](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L313) #### [Share lists deleted user as 'user'](https://github.com/owncloud/ocis/issues/903) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:676](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L676) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:677](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L677) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) -- [coreApiVersions/fileVersions.feature:499](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L499) //todo +- [coreApiVersions/fileVersions.feature:433](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L433) //todo #### [deleting a share with wrong authentication returns OCS status 996 / HTTP 500](https://github.com/owncloud/ocis/issues/1229) - [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:226](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L226) @@ -641,7 +641,7 @@ _ocs: api compatibility, return correct status code_ #### [Downloading the older version of shared file gives 404](https://github.com/owncloud/ocis/issues/3868) - [coreApiVersions/fileVersions.feature:160](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L160) - [coreApiVersions/fileVersions.feature:178](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L178) -- [coreApiVersions/fileVersions.feature:510](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L510) +- [coreApiVersions/fileVersions.feature:445](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L445) #### [file versions do not report the version author](https://github.com/owncloud/ocis/issues/2914) - [coreApiVersions/fileVersionAuthor.feature:12](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L12) diff --git a/tests/acceptance/expected-failures-on-S3NG-storage.md b/tests/acceptance/expected-failures-on-S3NG-storage.md index 15060169c0..57a3444281 100644 --- a/tests/acceptance/expected-failures-on-S3NG-storage.md +++ b/tests/acceptance/expected-failures-on-S3NG-storage.md @@ -12,7 +12,7 @@ Basic file management like up and download, move, copy, properties, quota, trash ### [Downloading the older version of shared file gives 404](https://github.com/owncloud/ocis/issues/3868) - [coreApiVersions/fileVersions.feature:160](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L160) - [coreApiVersions/fileVersions.feature:178](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L178) -- [coreApiVersions/fileVersions.feature:510](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L510) +- [coreApiVersions/fileVersions.feature:445](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L445) #### [file versions do not report the version author](https://github.com/owncloud/ocis/issues/2914) - [coreApiVersions/fileVersionAuthor.feature:12](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersionAuthor.feature#L12) @@ -321,11 +321,11 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiWebdavMove2/moveFile.feature:177](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiWebdavMove2/moveFile.feature#L177) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) //todo -- [coreApiVersions/fileVersions.feature:291](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L291) +- [coreApiVersions/fileVersions.feature:276](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L276) #### [restoring an older version of a shared file deletes the share](https://github.com/owncloud/ocis/issues/765) - [coreApiShareManagementToShares/acceptShares.feature:483](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementToShares/acceptShares.feature#L483) -- [coreApiVersions/fileVersions.feature:302](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L302) +- [coreApiVersions/fileVersions.feature:288](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L288) #### [Expiration date for shares is not implemented](https://github.com/owncloud/ocis/issues/1250) #### Expiration date of user shares @@ -415,14 +415,14 @@ _requires a [CS3 user provisioning api that can update the quota for a user](htt - [coreApiShareUpdateToShares/updateShare.feature:241](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareUpdateToShares/updateShare.feature#L241) #### [user can access version metadata of a received share before accepting it](https://github.com/owncloud/ocis/issues/760) -- [coreApiVersions/fileVersions.feature:487](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L487) +- [coreApiVersions/fileVersions.feature:313](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L313) #### [Share lists deleted user as 'user'](https://github.com/owncloud/ocis/issues/903) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:677](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L677) - [coreApiShareManagementBasicToShares/createShareToSharesFolder.feature:676](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/createShareToSharesFolder.feature#L676) #### [OCIS-storage overwriting a file as share receiver, does not create a new file version for the sharer](https://github.com/owncloud/ocis/issues/766) //todo -- [coreApiVersions/fileVersions.feature:499](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L499) +- [coreApiVersions/fileVersions.feature:433](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiVersions/fileVersions.feature#L433) #### [deleting a share with wrong authentication returns OCS status 996 / HTTP 500](https://github.com/owncloud/ocis/issues/1229) - [coreApiShareManagementBasicToShares/deleteShareFromShares.feature:226](https://github.com/owncloud/ocis/blob/master/tests/acceptance/features/coreApiShareManagementBasicToShares/deleteShareFromShares.feature#L226) diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index ddc885f1c3..f47eecc8c8 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -197,7 +197,7 @@ services: REVA_LDAP_HOSTNAME: ldap TEST_REVA: 'true' SEND_SCENARIO_LINE_REFERENCES: 'true' - BEHAT_FILTER_TAGS: ~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva + BEHAT_FILTER_TAGS: ~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster DIVIDE_INTO_NUM_PARTS: ${PARTS:-1} RUN_PART: ${PART:-1} EXPECTED_FAILURES_FILE: /mnt/acceptance/expected-failures-on-OCIS-storage.md @@ -266,7 +266,7 @@ services: REVA_LDAP_HOSTNAME: ldap TEST_REVA: 'true' SEND_SCENARIO_LINE_REFERENCES: 'true' - BEHAT_FILTER_TAGS: ~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva + BEHAT_FILTER_TAGS: ~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva&&~@skipOnRevaMaster DIVIDE_INTO_NUM_PARTS: ${PARTS:-1} RUN_PART: ${PART:-1} EXPECTED_FAILURES_FILE: /mnt/acceptance/expected-failures-on-S3NG-storage.md diff --git a/tests/ocis b/tests/ocis index 76cc388e45..4076bc71e6 160000 --- a/tests/ocis +++ b/tests/ocis @@ -1 +1 @@ -Subproject commit 76cc388e4546d4e588515c6c7d624829a674163a +Subproject commit 4076bc71e63b4dd4d7931e1ad085ee214f137e38 From fef69dfd5703518a60045a76901de8bfbf532417 Mon Sep 17 00:00:00 2001 From: Giuseppe Lo Presti Date: Fri, 28 Apr 2023 18:47:34 +0200 Subject: [PATCH 07/20] docs: updated examples for apps --- .../appprovider-codimd.toml | 1 - .../appprovider-collabora.toml | 19 +++++++++++++++++++ .../custom-mime-types-demo.json | 3 +++ examples/storage-references/gateway.toml | 11 ++++------- 4 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 examples/storage-references/appprovider-collabora.toml create mode 100644 examples/storage-references/custom-mime-types-demo.json diff --git a/examples/storage-references/appprovider-codimd.toml b/examples/storage-references/appprovider-codimd.toml index c5697f490c..da04af947a 100644 --- a/examples/storage-references/appprovider-codimd.toml +++ b/examples/storage-references/appprovider-codimd.toml @@ -17,4 +17,3 @@ wopi_url = "http://0.0.0.0:8880/" app_name = "CodiMD" app_url = "https://your-codimd-server.org:3000" app_int_url = "https://your-codimd-server.org:3000" -folder_base_url = "https://your-reva-frontend.org" diff --git a/examples/storage-references/appprovider-collabora.toml b/examples/storage-references/appprovider-collabora.toml new file mode 100644 index 0000000000..bade9bdfa9 --- /dev/null +++ b/examples/storage-references/appprovider-collabora.toml @@ -0,0 +1,19 @@ +[shared] +gatewaysvc = "localhost:your-revad-gateway-port" + +[grpc] +address = "0.0.0.0:12346" + +[grpc.services.appprovider] +driver = "wopi" +mime_types = ["application/vnd.oasis.opendocument.text", "application/vnd.oasis.opendocument.spreadsheet", "application/vnd.oasis.opendocument.presentation"] +app_provider_url = "localhost:12346" +language = "en-GB" + +[grpc.services.appprovider.drivers.wopi] +iop_secret = "hello" +wopi_url = "http://0.0.0.0:8880/" +app_name = "Collabora" +app_url = "https://your-collabora-server.org:9980" +app_int_url = "https://your-collabora-server.org:9980" +folder_base_url = "https://your-reva-frontend.org" diff --git a/examples/storage-references/custom-mime-types-demo.json b/examples/storage-references/custom-mime-types-demo.json new file mode 100644 index 0000000000..390947ec09 --- /dev/null +++ b/examples/storage-references/custom-mime-types-demo.json @@ -0,0 +1,3 @@ +{ + ".zmd": "application/compressed-markdown" +} diff --git a/examples/storage-references/gateway.toml b/examples/storage-references/gateway.toml index 098bc2325e..a8e90d8866 100644 --- a/examples/storage-references/gateway.toml +++ b/examples/storage-references/gateway.toml @@ -34,13 +34,10 @@ appauth = "localhost:15000" [grpc.services.appregistry.drivers.static] mime_types = [ {"mime_type" = "text/plain", "extension" = "txt", "name" = "Text file", "description" = "Text file", "allow_creation" = true}, - {"mime_type" = "text/markdown", "extension" = "md", "name" = "Markdown file", "description" = "Markdown file", "allow_creation" = true}, - {"mime_type" = "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "extension" = "docx", "name" = "Microsoft Word", "description" = "Microsoft Word document", "allow_creation" = true}, - {"mime_type" = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "extension" = "xlsx", "name" = "Microsoft Excel", "description" = "Microsoft Excel document", "allow_creation" = true}, - {"mime_type" = "application/vnd.openxmlformats-officedocument.presentationml.presentation", "extension" = "pptx", "name" = "Microsoft PowerPoint", "description" = "Microsoft PowerPoint document", "allow_creation" = true}, - {"mime_type" = "application/vnd.oasis.opendocument.text", "extension" = "odt", "name" = "OpenDocument", "description" = "OpenDocument text document", "allow_creation" = true}, - {"mime_type" = "application/vnd.oasis.opendocument.spreadsheet", "extension" = "ods", "name" = "OpenSpreadsheet", "description" = "OpenDocument spreadsheet document", "allow_creation" = true}, - {"mime_type" = "application/vnd.oasis.opendocument.presentation", "extension" = "odp", "name" = "OpenPresentation", "description" = "OpenDocument presentation document", "allow_creation" = true}, + {"mime_type" = "text/markdown", "extension" = "md", "name" = "Markdown file", "description" = "Markdown file", "default_app" = "CodiMD", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.text", "extension" = "odt", "name" = "OpenDocument", "description" = "OpenDocument text document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.spreadsheet", "extension" = "ods", "name" = "OpenSpreadsheet", "description" = "OpenDocument spreadsheet document", "default_app" = "Collabora", "allow_creation" = true}, + {"mime_type" = "application/vnd.oasis.opendocument.presentation", "extension" = "odp", "name" = "OpenPresentation", "description" = "OpenDocument presentation document", "default_app" = "Collabora", "allow_creation" = true}, {"mime_type" = "application/vnd.jupyter", "extension" = "ipynb", "name" = "Jupyter Notebook", "description" = "Jupyter Notebook"} ] From 4b04d7eeb56d4c274172742c1f37f6fdb7dd3393 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 2 May 2023 11:00:30 +0200 Subject: [PATCH 08/20] fix check permission on upload for single file share to lw account --- internal/grpc/interceptors/auth/scope.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/grpc/interceptors/auth/scope.go b/internal/grpc/interceptors/auth/scope.go index 63775539aa..b40e6125df 100644 --- a/internal/grpc/interceptors/auth/scope.go +++ b/internal/grpc/interceptors/auth/scope.go @@ -163,6 +163,11 @@ func checkLightweightScope(ctx context.Context, req interface{}, tokenScope map[ InitiateFileUpload: true, }) case *provider.InitiateFileUploadRequest: + if hasPermissions(ctx, client, r.GetRef(), &provider.ResourcePermissions{ + InitiateFileUpload: true, + }) { + return true + } parent, err := parentOfResource(ctx, client, r.GetRef()) if err != nil { return false From 1970b18569fbeca896148553bfaa4f76c51ef766 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 2 May 2023 15:07:19 +0200 Subject: [PATCH 09/20] add changelog --- .../unreleased/fix-upload-lw-accounts-single-file-share.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/unreleased/fix-upload-lw-accounts-single-file-share.md diff --git a/changelog/unreleased/fix-upload-lw-accounts-single-file-share.md b/changelog/unreleased/fix-upload-lw-accounts-single-file-share.md new file mode 100644 index 0000000000..46b447c49b --- /dev/null +++ b/changelog/unreleased/fix-upload-lw-accounts-single-file-share.md @@ -0,0 +1,3 @@ +Bugfix: Fix upload in a single file share for lightweight accounts + +https://github.com/cs3org/reva/pull/3838 From 5263e1c7483746406c5bf7ecdb22e2d64d700934 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Wed, 26 Apr 2023 16:39:22 +0200 Subject: [PATCH 10/20] refactored rest group driver --- pkg/cbox/group/rest/rest.go | 137 +++++++++++------------ pkg/cbox/user/rest/rest.go | 176 ++++++++++++++---------------- pkg/cbox/utils/tokenmanagement.go | 25 ++--- pkg/utils/list/list.go | 11 ++ 4 files changed, 164 insertions(+), 185 deletions(-) create mode 100644 pkg/utils/list/list.go diff --git a/pkg/cbox/group/rest/rest.go b/pkg/cbox/group/rest/rest.go index f2501677f1..6751d7d3c4 100644 --- a/pkg/cbox/group/rest/rest.go +++ b/pkg/cbox/group/rest/rest.go @@ -20,8 +20,8 @@ package rest import ( "context" - "errors" "fmt" + "net/url" "os" "os/signal" "strings" @@ -31,9 +31,11 @@ import ( grouppb "github.com/cs3org/go-cs3apis/cs3/identity/group/v1beta1" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/pkg/appctx" + user "github.com/cs3org/reva/pkg/cbox/user/rest" utils "github.com/cs3org/reva/pkg/cbox/utils" "github.com/cs3org/reva/pkg/group" "github.com/cs3org/reva/pkg/group/manager/registry" + "github.com/cs3org/reva/pkg/utils/list" "github.com/gomodule/redigo/redis" "github.com/mitchellh/mapstructure" "github.com/rs/zerolog/log" @@ -126,12 +128,12 @@ func New(m map[string]interface{}) (group.Manager, error) { redisPool: redisPool, apiTokenManager: apiTokenManager, } - go mgr.fetchAllGroups() + go mgr.fetchAllGroups(context.Background()) return mgr, nil } -func (m *manager) fetchAllGroups() { - _ = m.fetchAllGroupAccounts() +func (m *manager) fetchAllGroups(ctx context.Context) { + _ = m.fetchAllGroupAccounts(ctx) ticker := time.NewTicker(time.Duration(m.conf.GroupFetchInterval) * time.Second) work := make(chan os.Signal, 1) signal.Notify(work, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT) @@ -141,89 +143,73 @@ func (m *manager) fetchAllGroups() { case <-work: return case <-ticker.C: - _ = m.fetchAllGroupAccounts() + _ = m.fetchAllGroupAccounts(ctx) } } } -func (m *manager) fetchAllGroupAccounts() error { - ctx := context.Background() +type Group struct { + GroupIdentifier string `json:"groupIdentifier"` + DisplayName string `json:"displayName"` + Gid int `json:"gid,omitempty"` + IsComputingGroup bool `json:"isComputingGroup"` +} + +type GroupsResponse struct { + Pagination struct { + Links struct { + Next *string `json:"next"` + } `json:"links"` + } `json:"pagination"` + Data []*Group `json:"data"` +} + +func (m *manager) fetchAllGroupAccounts(ctx context.Context) error { url := fmt.Sprintf("%s/api/v1.0/Group?field=groupIdentifier&field=displayName&field=gid&field=isComputingGroup", m.conf.APIBaseURL) - for url != "" { - result, err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false) - if err != nil { + var r GroupsResponse + for { + if err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false, &r); err != nil { return err } - responseData, ok := result["data"].([]interface{}) - if !ok { - return errors.New("rest: error in type assertion") - } - for _, usr := range responseData { - groupData, ok := usr.(map[string]interface{}) - if !ok { - continue - } - - // filter computing groups - if is, ok := groupData["isComputingGroup"].(bool); ok && is { + for _, g := range r.Data { + if g.IsComputingGroup { continue } - - _, err = m.parseAndCacheGroup(ctx, groupData) - if err != nil { + if _, err := m.parseAndCacheGroup(ctx, g); err != nil { continue } } - url = "" - if pagination, ok := result["pagination"].(map[string]interface{}); ok { - if links, ok := pagination["links"].(map[string]interface{}); ok { - if next, ok := links["next"].(string); ok { - url = fmt.Sprintf("%s%s", m.conf.APIBaseURL, next) - } - } + if r.Pagination.Links.Next == nil { + break } + url = fmt.Sprintf("%s%s", m.conf.APIBaseURL, *r.Pagination.Links.Next) } return nil } -func (m *manager) parseAndCacheGroup(ctx context.Context, groupData map[string]interface{}) (*grouppb.Group, error) { - id, ok := groupData["groupIdentifier"].(string) - if !ok { - return nil, errors.New("rest: missing upn in user data") - } - - name, _ := groupData["displayName"].(string) +func (m *manager) parseAndCacheGroup(ctx context.Context, g *Group) (*grouppb.Group, error) { groupID := &grouppb.GroupId{ - OpaqueId: id, Idp: m.conf.IDProvider, + OpaqueId: g.GroupIdentifier, } - gid, ok := groupData["gid"].(int64) - if !ok { - gid = 0 - } - g := &grouppb.Group{ + + group := &grouppb.Group{ Id: groupID, - GroupName: id, - Mail: id + "@cern.ch", - DisplayName: name, - GidNumber: gid, + GroupName: g.GroupIdentifier, + Mail: g.GroupIdentifier + "@cern.ch", + DisplayName: g.DisplayName, + GidNumber: int64(g.Gid), } - if err := m.cacheGroupDetails(g); err != nil { + if err := m.cacheGroupDetails(group); err != nil { log.Error().Err(err).Msg("rest: error caching group details") } - if internalID, ok := groupData["id"].(string); ok { - if err := m.cacheInternalID(groupID, internalID); err != nil { - log.Error().Err(err).Msg("rest: error caching group details") - } - } - - return g, nil + return group, nil } func (m *manager) GetGroup(ctx context.Context, gid *grouppb.GroupId, skipFetchingMembers bool) (*grouppb.Group, error) { @@ -288,38 +274,39 @@ func (m *manager) GetMembers(ctx context.Context, gid *grouppb.GroupId) ([]*user return users, nil } - internalID, err := m.fetchCachedInternalID(gid) - if err != nil { - return nil, err - } - url := fmt.Sprintf("%s/api/v1.0/Group/%s/memberidentities/precomputed", m.conf.APIBaseURL, internalID) - result, err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false) + url, err := url.JoinPath(m.conf.APIBaseURL, "/api/v1.0/Group", gid.OpaqueId, "/memberidentities/precomputed?limit=10&field=upn&field=primaryAccountEmail&field=displayName&field=uid&field=gid&field=type&field=source") if err != nil { return nil, err } - userData := result["data"].([]interface{}) - users = []*userpb.UserId{} - - for _, u := range userData { - userInfo, ok := u.(map[string]interface{}) - if !ok { - return nil, errors.New("rest: error in type assertion") + var r user.IdentityResponse + members := []*userpb.UserId{} + for { + if err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false, &r); err != nil { + return nil, err } - if id, ok := userInfo["upn"].(string); ok { - users = append(users, &userpb.UserId{OpaqueId: id, Idp: m.conf.IDProvider}) + + users := list.Map(r.Data, func(i *user.Identity) *userpb.UserId { + return &userpb.UserId{OpaqueId: i.Upn, Idp: m.conf.IDProvider, Type: i.UserType()} + }) + members = append(members, users...) + + if r.Pagination.Links.Next == nil { + break } + url = fmt.Sprintf("%s%s", m.conf.APIBaseURL, *r.Pagination.Links.Next) } - if err = m.cacheGroupMembers(gid, users); err != nil { - log := appctx.GetLogger(ctx) - log.Error().Err(err).Msg("rest: error caching group members") + if err = m.cacheGroupMembers(gid, members); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Msg("rest: error caching group members") } return users, nil } func (m *manager) HasMember(ctx context.Context, gid *grouppb.GroupId, uid *userpb.UserId) (bool, error) { + // TODO (gdelmont): this can be improved storing the users a group is composed of as a list in redis + // and, instead of returning all the members, use the redis apis to check if the user is in the list. groupMemers, err := m.GetMembers(ctx, gid) if err != nil { return false, err diff --git a/pkg/cbox/user/rest/rest.go b/pkg/cbox/user/rest/rest.go index 443d86b0da..ca52f7dbf4 100644 --- a/pkg/cbox/user/rest/rest.go +++ b/pkg/cbox/user/rest/rest.go @@ -32,9 +32,9 @@ import ( utils "github.com/cs3org/reva/pkg/cbox/utils" "github.com/cs3org/reva/pkg/user" "github.com/cs3org/reva/pkg/user/manager/registry" + "github.com/cs3org/reva/pkg/utils/list" "github.com/gomodule/redigo/redis" "github.com/mitchellh/mapstructure" - "github.com/pkg/errors" "github.com/rs/zerolog/log" ) @@ -134,12 +134,12 @@ func (m *manager) Configure(ml map[string]interface{}) error { // Since we're starting a subroutine which would take some time to execute, // we can't wait to see if it works before returning the user.Manager object // TODO: return err if the fetch fails - go m.fetchAllUsers() + go m.fetchAllUsers(context.Background()) return nil } -func (m *manager) fetchAllUsers() { - _ = m.fetchAllUserAccounts() +func (m *manager) fetchAllUsers(ctx context.Context) { + _ = m.fetchAllUserAccounts(ctx) ticker := time.NewTicker(time.Duration(m.conf.UserFetchInterval) * time.Second) work := make(chan os.Signal, 1) signal.Notify(work, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT) @@ -149,79 +149,90 @@ func (m *manager) fetchAllUsers() { case <-work: return case <-ticker.C: - _ = m.fetchAllUserAccounts() + _ = m.fetchAllUserAccounts(ctx) } } } -func (m *manager) fetchAllUserAccounts() error { - ctx := context.Background() - url := fmt.Sprintf("%s/api/v1.0/Identity?field=upn&field=primaryAccountEmail&field=displayName&field=uid&field=gid&field=type", m.conf.APIBaseURL) +type Identity struct { + PrimaryAccountEmail string `json:"primaryAccountEmail,omitempty"` + Type string `json:"type,omitempty"` + Upn string `json:"upn"` + DisplayName string `json:"displayName"` + Source string `json:"source,omitempty"` + UID int `json:"uid,omitempty"` + GID int `json:"gid,omitempty"` +} - for url != "" { - result, err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false) - if err != nil { - return err +type IdentityResponse struct { + Pagination struct { + Links struct { + Next *string `json:"next"` + } `json:"links"` + } `json:"pagination"` + Data []*Identity `json:"data"` +} + +func (i *Identity) UserType() userpb.UserType { + switch i.Type { + case "Application": + return userpb.UserType_USER_TYPE_APPLICATION + case "Service": + return userpb.UserType_USER_TYPE_SERVICE + case "Secondary": + return userpb.UserType_USER_TYPE_SECONDARY + case "Person": + if i.Source == "cern" { + return userpb.UserType_USER_TYPE_PRIMARY } + return userpb.UserType_USER_TYPE_LIGHTWEIGHT + default: + return userpb.UserType_USER_TYPE_INVALID + } +} + +func (m *manager) fetchAllUserAccounts(ctx context.Context) error { + url := fmt.Sprintf("%s/api/v1.0/Identity?field=upn&field=primaryAccountEmail&field=displayName&field=uid&field=gid&field=type&field=source", m.conf.APIBaseURL) - responseData, ok := result["data"].([]interface{}) - if !ok { - return errors.New("rest: error in type assertion") + var r IdentityResponse + for { + if err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false, &r); err != nil { + return err } - for _, usr := range responseData { - userData, ok := usr.(map[string]interface{}) - if !ok { - continue - } - _, err = m.parseAndCacheUser(ctx, userData) - if err != nil { + for _, usr := range r.Data { + if _, err := m.parseAndCacheUser(ctx, usr); err != nil { continue } } - url = "" - if pagination, ok := result["pagination"].(map[string]interface{}); ok { - if links, ok := pagination["links"].(map[string]interface{}); ok { - if next, ok := links["next"].(string); ok { - url = fmt.Sprintf("%s%s", m.conf.APIBaseURL, next) - } - } + if r.Pagination.Links.Next == nil { + break } + url = fmt.Sprintf("%s%s", m.conf.APIBaseURL, *r.Pagination.Links.Next) } return nil } -func (m *manager) parseAndCacheUser(ctx context.Context, userData map[string]interface{}) (*userpb.User, error) { - upn, ok := userData["upn"].(string) - if !ok { - return nil, errors.New("rest: missing upn in user data") - } - mail, _ := userData["primaryAccountEmail"].(string) - name, _ := userData["displayName"].(string) - uidNumber, _ := userData["uid"].(float64) - gidNumber, _ := userData["gid"].(float64) - t, _ := userData["type"].(string) - userType := getUserType(t, upn) - - userID := &userpb.UserId{ - OpaqueId: upn, - Idp: m.conf.IDProvider, - Type: userType, - } +func (m *manager) parseAndCacheUser(ctx context.Context, i *Identity) (*userpb.User, error) { u := &userpb.User{ - Id: userID, - Username: upn, - Mail: mail, - DisplayName: name, - UidNumber: int64(uidNumber), - GidNumber: int64(gidNumber), + Id: &userpb.UserId{ + OpaqueId: i.Upn, + Idp: m.conf.IDProvider, + Type: i.UserType(), + }, + Username: i.Upn, + Mail: i.PrimaryAccountEmail, + DisplayName: i.DisplayName, + UidNumber: int64(i.UID), + GidNumber: int64(i.GID), } if err := m.cacheUserDetails(u); err != nil { log.Error().Err(err).Msg("rest: error caching user details") } + return u, nil } @@ -309,31 +320,34 @@ func isUserAnyType(user *userpb.User, types []userpb.UserType) bool { return false } +type Group struct { + DisplayName string `json:"displayName"` +} + +type GroupsResponse struct { + Pagination struct { + Links struct { + Next *string `json:"next"` + } `json:"links"` + } `json:"pagination"` + Data []Group `json:"data"` +} + func (m *manager) GetUserGroups(ctx context.Context, uid *userpb.UserId) ([]string, error) { groups, err := m.fetchCachedUserGroups(uid) if err == nil { return groups, nil } - url := fmt.Sprintf("%s/api/v1.0/Identity/%s/groups?recursive=true", m.conf.APIBaseURL, uid.OpaqueId) - result, err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false) - if err != nil { + // TODO (gdelmont): support pagination! we may have problems with users having more than 1000 groups + url := fmt.Sprintf("%s/api/v1.0/Identity/%s/groups?field=displayName&recursive=true", m.conf.APIBaseURL, uid.OpaqueId) + + var r GroupsResponse + if err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false, &r); err != nil { return nil, err } - groupData := result["data"].([]interface{}) - groups = []string{} - - for _, g := range groupData { - groupInfo, ok := g.(map[string]interface{}) - if !ok { - return nil, errors.New("rest: error in type assertion") - } - name, ok := groupInfo["displayName"].(string) - if ok { - groups = append(groups, name) - } - } + groups = list.Map(r.Data, func(g Group) string { return g.DisplayName }) if err = m.cacheUserGroups(uid, groups); err != nil { log := appctx.GetLogger(ctx) @@ -344,6 +358,8 @@ func (m *manager) GetUserGroups(ctx context.Context, uid *userpb.UserId) ([]stri } func (m *manager) IsInGroup(ctx context.Context, uid *userpb.UserId, group string) (bool, error) { + // TODO (gdelmont): this can be improved storing the groups a user belong to as a list in redis + // and, instead of returning all the groups, use the redis apis to check if the group is in the list. userGroups, err := m.GetUserGroups(ctx, uid) if err != nil { return false, err @@ -356,27 +372,3 @@ func (m *manager) IsInGroup(ctx context.Context, uid *userpb.UserId, group strin } return false, nil } - -func getUserType(userType, upn string) userpb.UserType { - var t userpb.UserType - switch userType { - case "Application": - t = userpb.UserType_USER_TYPE_APPLICATION - case "Service": - t = userpb.UserType_USER_TYPE_SERVICE - case "Secondary": - t = userpb.UserType_USER_TYPE_SECONDARY - case "Person": - switch { - case strings.HasPrefix(upn, "guest"): - t = userpb.UserType_USER_TYPE_LIGHTWEIGHT - case strings.Contains(upn, "@"): - t = userpb.UserType_USER_TYPE_FEDERATED - default: - t = userpb.UserType_USER_TYPE_PRIMARY - } - default: - t = userpb.UserType_USER_TYPE_INVALID - } - return t -} diff --git a/pkg/cbox/utils/tokenmanagement.go b/pkg/cbox/utils/tokenmanagement.go index 19d78924dc..6eea766d92 100644 --- a/pkg/cbox/utils/tokenmanagement.go +++ b/pkg/cbox/utils/tokenmanagement.go @@ -128,15 +128,15 @@ func (a *APITokenManager) getAPIToken(ctx context.Context) (string, time.Time, e } // SendAPIGetRequest makes an API GET Request to the passed URL. -func (a *APITokenManager) SendAPIGetRequest(ctx context.Context, url string, forceRenewal bool) (map[string]interface{}, error) { +func (a *APITokenManager) SendAPIGetRequest(ctx context.Context, url string, forceRenewal bool, v any) error { err := a.renewAPIToken(ctx, forceRenewal) if err != nil { - return nil, err + return err } httpReq, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { - return nil, err + return err } // We don't need to take the lock when reading apiToken, because if we reach here, @@ -146,28 +146,17 @@ func (a *APITokenManager) SendAPIGetRequest(ctx context.Context, url string, for httpRes, err := a.client.Do(httpReq) if err != nil { - return nil, err + return err } defer httpRes.Body.Close() if httpRes.StatusCode == http.StatusUnauthorized { // The token is no longer valid, try renewing it - return a.SendAPIGetRequest(ctx, url, true) + return a.SendAPIGetRequest(ctx, url, true, v) } if httpRes.StatusCode < 200 || httpRes.StatusCode > 299 { - return nil, errors.New("rest: API request returned " + httpRes.Status) - } - - body, err := io.ReadAll(httpRes.Body) - if err != nil { - return nil, err - } - - var result map[string]interface{} - err = json.Unmarshal(body, &result) - if err != nil { - return nil, err + return errors.New("rest: API request returned " + httpRes.Status) } - return result, nil + return json.NewDecoder(httpRes.Body).Decode(v) } diff --git a/pkg/utils/list/list.go b/pkg/utils/list/list.go new file mode 100644 index 0000000000..433a64ef6a --- /dev/null +++ b/pkg/utils/list/list.go @@ -0,0 +1,11 @@ +package list + +// Map returns a list constructed by appling a function f +// to all items in the list l. +func Map[T, V any](l []T, f func(T) V) []V { + m := make([]V, 0, len(l)) + for _, e := range l { + m = append(m, f(e)) + } + return m +} From d5a74d82809b5c49b8ddd1150fc6e454ae8e7b23 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 2 May 2023 14:54:23 +0200 Subject: [PATCH 11/20] fix linter --- pkg/cbox/group/rest/rest.go | 5 ++++- pkg/cbox/user/rest/rest.go | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/cbox/group/rest/rest.go b/pkg/cbox/group/rest/rest.go index 6751d7d3c4..f32d7ba860 100644 --- a/pkg/cbox/group/rest/rest.go +++ b/pkg/cbox/group/rest/rest.go @@ -148,6 +148,7 @@ func (m *manager) fetchAllGroups(ctx context.Context) { } } +// Group contains the information about a group. type Group struct { GroupIdentifier string `json:"groupIdentifier"` DisplayName string `json:"displayName"` @@ -155,6 +156,8 @@ type Group struct { IsComputingGroup bool `json:"isComputingGroup"` } +// GroupsResponse contains the expected response from grappa +// when getting the list of groups. type GroupsResponse struct { Pagination struct { Links struct { @@ -279,7 +282,7 @@ func (m *manager) GetMembers(ctx context.Context, gid *grouppb.GroupId) ([]*user return nil, err } - var r user.IdentityResponse + var r user.IdentitiesResponse members := []*userpb.UserId{} for { if err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false, &r); err != nil { diff --git a/pkg/cbox/user/rest/rest.go b/pkg/cbox/user/rest/rest.go index ca52f7dbf4..a3e1de1a23 100644 --- a/pkg/cbox/user/rest/rest.go +++ b/pkg/cbox/user/rest/rest.go @@ -154,6 +154,7 @@ func (m *manager) fetchAllUsers(ctx context.Context) { } } +// Identity contains the information of a single user. type Identity struct { PrimaryAccountEmail string `json:"primaryAccountEmail,omitempty"` Type string `json:"type,omitempty"` @@ -164,7 +165,9 @@ type Identity struct { GID int `json:"gid,omitempty"` } -type IdentityResponse struct { +// IdentitiesResponse contains the expected response from grappa +// when getting the list of users. +type IdentitiesResponse struct { Pagination struct { Links struct { Next *string `json:"next"` @@ -173,6 +176,7 @@ type IdentityResponse struct { Data []*Identity `json:"data"` } +// UserType convert the user type in grappa to CS3APIs. func (i *Identity) UserType() userpb.UserType { switch i.Type { case "Application": @@ -194,7 +198,7 @@ func (i *Identity) UserType() userpb.UserType { func (m *manager) fetchAllUserAccounts(ctx context.Context) error { url := fmt.Sprintf("%s/api/v1.0/Identity?field=upn&field=primaryAccountEmail&field=displayName&field=uid&field=gid&field=type&field=source", m.conf.APIBaseURL) - var r IdentityResponse + var r IdentitiesResponse for { if err := m.apiTokenManager.SendAPIGetRequest(ctx, url, false, &r); err != nil { return err @@ -320,10 +324,13 @@ func isUserAnyType(user *userpb.User, types []userpb.UserType) bool { return false } +// Group contains the information about a group. type Group struct { DisplayName string `json:"displayName"` } +// GroupsResponse contains the expected response from grappa +// when getting the list of groups. type GroupsResponse struct { Pagination struct { Links struct { From fe33097bfd6d90b0065741b5668e5a6cf00239f8 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 2 May 2023 15:01:29 +0200 Subject: [PATCH 12/20] add changelog --- .../unreleased/revamp-rest-used-group-drivers.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/revamp-rest-used-group-drivers.md diff --git a/changelog/unreleased/revamp-rest-used-group-drivers.md b/changelog/unreleased/revamp-rest-used-group-drivers.md new file mode 100644 index 0000000000..f841f6f8d9 --- /dev/null +++ b/changelog/unreleased/revamp-rest-used-group-drivers.md @@ -0,0 +1,11 @@ +Enhancement: Revamp user/group drivers and fix user type +for lightweight accounts + +* Fix the user type for lightweight accounts, using the +source field to differentiate between a primary and lw account +* Remove all the code with manual parsing of the json returned +by the CERN provider +* Introduce pagination for `GetMembers` method in the group driver +* Reduced network transfer size by requesting only needed fields for `GetMembers` method + +https://github.com/cs3org/reva/pull/3821 From fec220a1e4430dec957efafd27e13161b3dde9f5 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 2 May 2023 15:03:31 +0200 Subject: [PATCH 13/20] removed unused methods --- pkg/cbox/group/rest/cache.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pkg/cbox/group/rest/cache.go b/pkg/cbox/group/rest/cache.go index 84723bd610..2f2baf429b 100644 --- a/pkg/cbox/group/rest/cache.go +++ b/pkg/cbox/group/rest/cache.go @@ -106,14 +106,6 @@ func (m *manager) getVal(key string) (string, error) { return "", errors.New("rest: unable to get connection from redis pool") } -func (m *manager) fetchCachedInternalID(gid *grouppb.GroupId) (string, error) { - return m.getVal(groupPrefix + groupInternalIDPrefix + gid.OpaqueId) -} - -func (m *manager) cacheInternalID(gid *grouppb.GroupId, internalID string) error { - return m.setVal(groupPrefix+groupInternalIDPrefix+gid.OpaqueId, internalID, -1) -} - func (m *manager) findCachedGroups(query string) ([]*grouppb.Group, error) { conn := m.redisPool.Get() defer conn.Close() From 13326f4474f53fd99e3adb976f2be4ef1d4f3eb4 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 2 May 2023 15:09:24 +0200 Subject: [PATCH 14/20] fix field name --- pkg/cbox/group/rest/rest.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cbox/group/rest/rest.go b/pkg/cbox/group/rest/rest.go index f32d7ba860..e7a8cc8bc2 100644 --- a/pkg/cbox/group/rest/rest.go +++ b/pkg/cbox/group/rest/rest.go @@ -152,7 +152,7 @@ func (m *manager) fetchAllGroups(ctx context.Context) { type Group struct { GroupIdentifier string `json:"groupIdentifier"` DisplayName string `json:"displayName"` - Gid int `json:"gid,omitempty"` + GID int `json:"gid,omitempty"` IsComputingGroup bool `json:"isComputingGroup"` } @@ -205,7 +205,7 @@ func (m *manager) parseAndCacheGroup(ctx context.Context, g *Group) (*grouppb.Gr GroupName: g.GroupIdentifier, Mail: g.GroupIdentifier + "@cern.ch", DisplayName: g.DisplayName, - GidNumber: int64(g.Gid), + GidNumber: int64(g.GID), } if err := m.cacheGroupDetails(group); err != nil { From 57cec27590112c686baee9d3fdc7f4b5f5de2a84 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 2 May 2023 17:16:45 +0200 Subject: [PATCH 15/20] add header --- pkg/utils/list/list.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/utils/list/list.go b/pkg/utils/list/list.go index 433a64ef6a..ba83b2de86 100644 --- a/pkg/utils/list/list.go +++ b/pkg/utils/list/list.go @@ -1,3 +1,21 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + package list // Map returns a list constructed by appling a function f From c3860976b2364f66406778ab68505e7693ebb109 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Wed, 3 May 2023 18:41:31 +0200 Subject: [PATCH 16/20] Support multiple issuers in OIDC driver (#3839) * support multiple issuers in the oidc provider * fix init provider with correct issuer * add changelog * fix lint * trigger ci --- changelog/unreleased/multiple-issuers-oidc.md | 9 + go.mod | 5 +- go.sum | 13 + pkg/auth/manager/oidc/oidc.go | 298 +++++++----------- 4 files changed, 138 insertions(+), 187 deletions(-) create mode 100644 changelog/unreleased/multiple-issuers-oidc.md diff --git a/changelog/unreleased/multiple-issuers-oidc.md b/changelog/unreleased/multiple-issuers-oidc.md new file mode 100644 index 0000000000..5c12390bbc --- /dev/null +++ b/changelog/unreleased/multiple-issuers-oidc.md @@ -0,0 +1,9 @@ +Enhancement: Support multiple issuer in OIDC auth driver + +The OIDC auth driver supports now multiple issuers. Users of +external providers are then mapped to a local user by a +mapping files. Only the main issuer (defined in the config +with `issuer`) and the ones defined in the mapping are +allowed for the verification of the OIDC token. + +https://github.com/cs3org/reva/pull/3839 \ No newline at end of file diff --git a/go.mod b/go.mod index cdc0d08bfb..3649a5d430 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( go.opentelemetry.io/otel/trace v1.11.2 go.step.sm/crypto v0.23.2 golang.org/x/crypto v0.5.0 - golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 + golang.org/x/oauth2 v0.3.0 golang.org/x/sync v0.1.0 golang.org/x/sys v0.5.0 golang.org/x/term v0.5.0 @@ -77,6 +77,8 @@ require ( gotest.tools v2.2.0+incompatible ) +require github.com/go-jose/go-jose/v3 v3.0.0 // indirect + require ( github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -87,6 +89,7 @@ require ( github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/coreos/go-oidc/v3 v3.5.0 github.com/davecgh/go-spew v1.1.1 // indirect github.com/dolthub/vitess v0.0.0-20221031111135-9aad77e7b39f // indirect github.com/dustin/go-humanize v1.0.0 // indirect diff --git a/go.sum b/go.sum index 0e1f467108..3959aff6e3 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,7 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= @@ -289,6 +290,8 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-oidc/v3 v3.5.0 h1:VxKtbccHZxs8juq7RdJntSqtXFtde9YpNpGn0yqgEHw= +github.com/coreos/go-oidc/v3 v3.5.0/go.mod h1:ecXRtV4romGPeO6ieExAsUK9cb/3fp9hXNz1tlv8PIM= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -393,6 +396,8 @@ github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= @@ -1234,6 +1239,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1364,6 +1370,8 @@ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1391,6 +1399,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1530,6 +1540,7 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1537,6 +1548,7 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1549,6 +1561,7 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/auth/manager/oidc/oidc.go b/pkg/auth/manager/oidc/oidc.go index 6d8169e118..d55b6cd420 100644 --- a/pkg/auth/manager/oidc/oidc.go +++ b/pkg/auth/manager/oidc/oidc.go @@ -28,7 +28,7 @@ import ( "strings" "time" - oidc "github.com/coreos/go-oidc" + "github.com/coreos/go-oidc/v3/oidc" authpb "github.com/cs3org/go-cs3apis/cs3/auth/provider/v1beta1" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -41,6 +41,7 @@ import ( "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/rhttp" "github.com/cs3org/reva/pkg/sharedconf" + "github.com/golang-jwt/jwt" "github.com/juliangruber/go-intersect" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" @@ -52,7 +53,8 @@ func init() { } type mgr struct { - provider *oidc.Provider // cached on first request + providers map[string]*oidc.Provider + c *config oidcUsersMapping map[string]*oidcUserMapping } @@ -103,7 +105,9 @@ func parseConfig(m map[string]interface{}) (*config, error) { // New returns an auth manager implementation that verifies the oidc token and obtains the user claims. func New(m map[string]interface{}) (auth.Manager, error) { - manager := &mgr{} + manager := &mgr{ + providers: make(map[string]*oidc.Provider), + } err := manager.Configure(m) if err != nil { return nil, err @@ -144,103 +148,147 @@ func (am *mgr) Configure(m map[string]interface{}) error { return nil } -// The clientID would be empty as we only need to validate the clientSecret variable -// which contains the access token that we can use to contact the UserInfo endpoint -// and get the user claims. -func (am *mgr) Authenticate(ctx context.Context, _, clientSecret string) (*user.User, map[string]*authpb.Scope, error) { - ctx = am.getOAuthCtx(ctx) - log := appctx.GetLogger(ctx) - - oidcProvider, err := am.getOIDCProvider(ctx) +func extractClaims(token string) (jwt.MapClaims, error) { + var claims jwt.MapClaims + _, _, err := new(jwt.Parser).ParseUnverified(token, &claims) if err != nil { - return nil, nil, fmt.Errorf("oidc: error creating oidc provider: +%v", err) + return nil, err } + return claims, nil +} - oauth2Token := &oauth2.Token{ - AccessToken: clientSecret, +func extractIssuer(m jwt.MapClaims) (string, bool) { + issIface, ok := m["iss"] + if !ok { + return "", false } + iss, _ := issIface.(string) + return iss, iss != "" +} - // query the oidc provider for user info - userInfo, err := oidcProvider.UserInfo(ctx, oauth2.StaticTokenSource(oauth2Token)) - if err != nil { - return nil, nil, fmt.Errorf("oidc: error getting userinfo: +%v", err) +func (am *mgr) getOIDCProviderForIssuer(ctx context.Context, issuer string) (*oidc.Provider, error) { + // FIXME: op not atomic TODO: fix message and make it more clear + if am.providers[issuer] == nil { + // TODO (gdelmont): the provider should be periodically recreated + // as the public key can change over time + provider, err := oidc.NewProvider(ctx, issuer) + if err != nil { + return nil, errors.Wrapf(err, "oidc: error creating a new oidc provider") + } + am.providers[issuer] = provider } + return am.providers[issuer], nil +} - // claims contains the standard OIDC claims like iss, iat, aud, ... and any other non-standard one. - // TODO(labkode): make claims configuration dynamic from the config file so we can add arbitrary mappings from claims to user struct. - // For now, only the group claim is dynamic. - // TODO(labkode): may do like K8s does it: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/plugin/pkg/authenticator/token/oidc/oidc.go - var claims map[string]interface{} - if err := userInfo.Claims(&claims); err != nil { - return nil, nil, fmt.Errorf("oidc: error unmarshaling userinfo claims: %v", err) +func (am *mgr) isIssuerAllowed(issuer string) bool { + if am.c.Issuer == issuer { + return true } + for _, m := range am.oidcUsersMapping { + if m.OIDCIssuer == issuer { + return true + } + } + return false +} - log.Debug().Interface("claims", claims).Interface("userInfo", userInfo).Msg("unmarshalled userinfo") +func (am *mgr) doUserMapping(tkn *oidc.IDToken, claims jwt.MapClaims) (string, error) { + if len(am.oidcUsersMapping) == 0 { + return tkn.Subject, nil + } + // we need the custom claims for the mapping + if claims[am.c.GroupClaim] == nil { + // we are required to perform a user mapping but the group claim is not available + return tkn.Subject, nil + } - if claims["iss"] == nil { // This is not set in simplesamlphp - claims["iss"] = am.c.Issuer + mappings := make([]string, 0, len(am.oidcUsersMapping)) + for _, m := range am.oidcUsersMapping { + if m.OIDCIssuer == tkn.Issuer { + mappings = append(mappings, m.OIDCGroup) + } } - if claims["email_verified"] == nil { // This is not set in simplesamlphp - claims["email_verified"] = false + + intersection := intersect.Simple(claims[am.c.GroupClaim], mappings) + if len(intersection) > 1 { + // multiple mappings are not implemented as we cannot decide which one to choose + return "", errtypes.PermissionDenied("more than one user mapping entry exists for the given group claims") } - if claims["preferred_username"] == nil { - claims["preferred_username"] = claims[am.c.IDClaim] + if len(intersection) == 0 { + return "", errtypes.PermissionDenied("no user mapping found for the given group claim(s)") } - if claims["preferred_username"] == nil { - claims["preferred_username"] = claims["email"] + m := intersection[0].(string) + return am.oidcUsersMapping[m].Username, nil +} + +// The clientID would be empty as we only need to validate the clientSecret variable +// which contains the access token that we can use to contact the UserInfo endpoint +// and get the user claims. +func (am *mgr) Authenticate(ctx context.Context, _, clientSecret string) (*user.User, map[string]*authpb.Scope, error) { + log := appctx.GetLogger(ctx) + ctx = am.getOAuthCtx(ctx) + + claims, err := extractClaims(clientSecret) + if err != nil { + return nil, nil, errtypes.PermissionDenied("oidc token not valid") } - if claims["name"] == nil { - claims["name"] = claims[am.c.IDClaim] + + issuer, ok := extractIssuer(claims) + if !ok { + return nil, nil, errtypes.PermissionDenied("issuer not contained in the token") + } + log.Debug().Str("issuer", issuer).Msg("extracted issuer from token") + + if !am.isIssuerAllowed(issuer) { + log.Debug().Str("issuer", issuer).Msg("issuer is not in the whitelist") + return nil, nil, errtypes.PermissionDenied("issuer not recognised") } - if claims["name"] == nil { - return nil, nil, fmt.Errorf("no \"name\" attribute found in userinfo: maybe the client did not request the oidc \"profile\"-scope") + log.Debug().Str("issuer", issuer).Msg("issuer is whitelisted") + + provider, err := am.getOIDCProviderForIssuer(ctx, issuer) + if err != nil { + return nil, nil, errors.Wrap(err, "oidc: error creating oidc provider") } - if claims["email"] == nil { - return nil, nil, fmt.Errorf("no \"email\" attribute found in userinfo: maybe the client did not request the oidc \"email\"-scope") + + config := &oidc.Config{ + SkipClientIDCheck: true, } - err = am.resolveUser(ctx, claims, userInfo.Subject) + tkn, err := provider.Verifier(config).Verify(ctx, clientSecret) if err != nil { - return nil, nil, errors.Wrapf(err, "oidc: error resolving username for external user '%v'", claims["email"]) + return nil, nil, errtypes.PermissionDenied(fmt.Sprintf("oidc token not valid: %+v", err)) } - userID := &user.UserId{ - OpaqueId: claims[am.c.IDClaim].(string), // a stable non reassignable id - Idp: claims["iss"].(string), // in the scope of this issuer - Type: getUserType(claims[am.c.IDClaim].(string)), + sub, err := am.doUserMapping(tkn, claims) + if err != nil { + return nil, nil, err } + log.Debug().Str("sub", sub).Msg("mapped user from token") - gwc, err := pool.GetGatewayServiceClient(pool.Endpoint(am.c.GatewaySvc)) + client, err := pool.GetGatewayServiceClient(pool.Endpoint(am.c.GatewaySvc)) if err != nil { - return nil, nil, errors.Wrap(err, "oidc: error getting gateway grpc client") + return nil, nil, errors.Wrap(err, "error getting user provider grpc client") } - getGroupsResp, err := gwc.GetUserGroups(ctx, &user.GetUserGroupsRequest{ - UserId: userID, + userRes, err := client.GetUserByClaim(ctx, &user.GetUserByClaimRequest{ + Claim: "username", + Value: sub, }) if err != nil { - return nil, nil, errors.Wrapf(err, "oidc: error getting user groups for '%+v'", userID) + return nil, nil, errors.Wrapf(err, "error getting user by username '%v'", sub) } - if getGroupsResp.Status.Code != rpc.Code_CODE_OK { - return nil, nil, status.NewErrorFromCode(getGroupsResp.Status.Code, "oidc") + if userRes.Status.Code != rpc.Code_CODE_OK { + return nil, nil, status.NewErrorFromCode(userRes.Status.Code, "oidc") } - u := &user.User{ - Id: userID, - Username: claims["preferred_username"].(string), - Groups: getGroupsResp.Groups, - Mail: claims["email"].(string), - MailVerified: claims["email_verified"].(bool), - DisplayName: claims["name"].(string), - UidNumber: claims[am.c.UIDClaim].(int64), - GidNumber: claims[am.c.GIDClaim].(int64), - } + u := userRes.GetUser() var scopes map[string]*authpb.Scope - if userID != nil && (userID.Type == user.UserType_USER_TYPE_LIGHTWEIGHT || userID.Type == user.UserType_USER_TYPE_FEDERATED) { + if u.Id.Type == user.UserType_USER_TYPE_LIGHTWEIGHT { scopes, err = scope.AddLightweightAccountScope(authpb.Role_ROLE_OWNER, nil) if err != nil { return nil, nil, err } + // TODO (gdelmont): we may want to define a template to prettify the user info for lw account? // strip the `guest:` prefix if present in the email claim (appears to come from LDAP at CERN?) u.Mail = strings.Replace(u.Mail, "guest: ", "", 1) // and decorate the display name with the email domain to make it different from a primary account @@ -255,15 +303,6 @@ func (am *mgr) Authenticate(ctx context.Context, _, clientSecret string) (*user. return u, scopes, nil } -func (am *mgr) getUserID(claims map[string]interface{}) (int64, int64) { - uidf, _ := claims[am.c.UIDClaim].(float64) - uid := int64(uidf) - - gidf, _ := claims[am.c.GIDClaim].(float64) - gid := int64(gidf) - return uid, gid -} - func (am *mgr) getOAuthCtx(ctx context.Context) context.Context { // Sometimes for testing we need to skip the TLS check, that's why we need a // custom HTTP client. @@ -277,116 +316,3 @@ func (am *mgr) getOAuthCtx(ctx context.Context) context.Context { ctx = context.WithValue(ctx, oauth2.HTTPClient, customHTTPClient) return ctx } - -// getOIDCProvider returns a singleton OIDC provider. -func (am *mgr) getOIDCProvider(ctx context.Context) (*oidc.Provider, error) { - ctx = am.getOAuthCtx(ctx) - log := appctx.GetLogger(ctx) - - if am.provider != nil { - return am.provider, nil - } - - // Initialize a provider by specifying the issuer URL. - // Once initialized this is a singleton that is reused for further requests. - // The provider is responsible to verify the token sent by the client - // against the security keys oftentimes available in the .well-known endpoint. - provider, err := oidc.NewProvider(ctx, am.c.Issuer) - - if err != nil { - log.Error().Err(err).Msg("oidc: error creating a new oidc provider") - return nil, fmt.Errorf("oidc: error creating a new oidc provider: %+v", err) - } - - am.provider = provider - return am.provider, nil -} - -func (am *mgr) resolveUser(ctx context.Context, claims map[string]interface{}, subject string) error { - var ( - value string - resolve bool - ) - - uid, gid := am.getUserID(claims) - if uid != 0 && gid != 0 { - claims[am.c.UIDClaim] = uid - claims[am.c.GIDClaim] = gid - } - - if len(am.oidcUsersMapping) > 0 { - // map and discover the user's username when a mapping is defined - if claims[am.c.GroupClaim] == nil { - // we are required to perform a user mapping but the group claim is not available - return fmt.Errorf("no \"%s\" claim found in userinfo to map user", am.c.GroupClaim) - } - mappings := make([]string, 0, len(am.oidcUsersMapping)) - for _, m := range am.oidcUsersMapping { - if m.OIDCIssuer == claims["iss"] { - mappings = append(mappings, m.OIDCGroup) - } - } - - intersection := intersect.Simple(claims[am.c.GroupClaim], mappings) - if len(intersection) > 1 { - // multiple mappings are not implemented as we cannot decide which one to choose - return errtypes.PermissionDenied("more than one user mapping entry exists for the given group claims") - } - if len(intersection) == 0 { - return errtypes.PermissionDenied("no user mapping found for the given group claim(s)") - } - for _, m := range intersection { - value = am.oidcUsersMapping[m.(string)].Username - } - resolve = true - } else if uid == 0 || gid == 0 { - value = subject - resolve = true - } - - if !resolve { - return nil - } - - upsc, err := pool.GetGatewayServiceClient(pool.Endpoint(am.c.GatewaySvc)) - if err != nil { - return errors.Wrap(err, "error getting user provider grpc client") - } - getUserByClaimResp, err := upsc.GetUserByClaim(ctx, &user.GetUserByClaimRequest{ - Claim: "username", - Value: value, - }) - if err != nil { - return errors.Wrapf(err, "error getting user by username '%v'", value) - } - if getUserByClaimResp.Status.Code != rpc.Code_CODE_OK { - return status.NewErrorFromCode(getUserByClaimResp.Status.Code, "oidc") - } - - // take the properties of the mapped target user to override the claims - claims["preferred_username"] = getUserByClaimResp.GetUser().Username - claims[am.c.IDClaim] = getUserByClaimResp.GetUser().GetId().OpaqueId - claims["iss"] = getUserByClaimResp.GetUser().GetId().Idp - claims[am.c.UIDClaim] = getUserByClaimResp.GetUser().UidNumber - claims[am.c.GIDClaim] = getUserByClaimResp.GetUser().GidNumber - log := appctx.GetLogger(ctx).Debug().Str("username", value).Interface("claims", claims) - if uid == 0 || gid == 0 { - log.Msgf("resolveUser: claims overridden from '%s'", subject) - } else { - log.Msg("resolveUser: claims overridden from mapped user") - } - return nil -} - -func getUserType(upn string) user.UserType { - var t user.UserType - switch { - case strings.HasPrefix(upn, "guest"): - t = user.UserType_USER_TYPE_LIGHTWEIGHT - case strings.Contains(upn, "@"): - t = user.UserType_USER_TYPE_FEDERATED - default: - t = user.UserType_USER_TYPE_PRIMARY - } - return t -} From d06ee85eb5c4f6ecc2db9ab3d7fc7b68731576b8 Mon Sep 17 00:00:00 2001 From: Antoon P Date: Thu, 4 May 2023 14:20:17 +0200 Subject: [PATCH 17/20] Update the model before persisting (#3749) Co-authored-by: Antoon P --- changelog/unreleased/fix-3747.md | 3 +++ pkg/ocm/share/repository/json/json.go | 1 + 2 files changed, 4 insertions(+) create mode 100644 changelog/unreleased/fix-3747.md diff --git a/changelog/unreleased/fix-3747.md b/changelog/unreleased/fix-3747.md new file mode 100644 index 0000000000..7b7ad98503 --- /dev/null +++ b/changelog/unreleased/fix-3747.md @@ -0,0 +1,3 @@ +Bugfix: Fix persisting updates of received shares in json driver + +https://github.com/cs3org/reva/pull/3749 \ No newline at end of file diff --git a/pkg/ocm/share/repository/json/json.go b/pkg/ocm/share/repository/json/json.go index defedb3440..109a3d367e 100644 --- a/pkg/ocm/share/repository/json/json.go +++ b/pkg/ocm/share/repository/json/json.go @@ -508,6 +508,7 @@ func (m *mgr) UpdateReceivedShare(ctx context.Context, user *userpb.User, share switch mask { case "state": rs.State = share.State + m.model.ReceivedShares[share.Id.OpaqueId].State = share.State // TODO case "mount_point": default: return nil, errtypes.NotSupported("updating " + mask + " is not supported") From 8957d89151b52e19184c31645ca01ccd826fdc0e Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte <39946305+gmgigi96@users.noreply.github.com> Date: Thu, 4 May 2023 18:28:18 +0200 Subject: [PATCH 18/20] Create OCM share from sciencemesh service (#3695) * create share from sciencemesh service * make /create-share unprotected * add changelog * fix linter! * Update internal/http/services/sciencemesh/share.go Co-authored-by: Giuseppe Lo Presti * Update internal/http/services/sciencemesh/share.go Co-authored-by: Giuseppe Lo Presti * Update internal/http/services/sciencemesh/sciencemesh.go Co-authored-by: Giuseppe Lo Presti * removed old code * fix import names --------- Co-authored-by: Giuseppe Lo Presti --- changelog/unreleased/ocm-share-create-sm.md | 4 + .../http/services/sciencemesh/sciencemesh.go | 6 +- internal/http/services/sciencemesh/share.go | 176 ++++++++++++++++++ 3 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/ocm-share-create-sm.md create mode 100644 internal/http/services/sciencemesh/share.go diff --git a/changelog/unreleased/ocm-share-create-sm.md b/changelog/unreleased/ocm-share-create-sm.md new file mode 100644 index 0000000000..ac2e53e05f --- /dev/null +++ b/changelog/unreleased/ocm-share-create-sm.md @@ -0,0 +1,4 @@ +Enhancement: Create OCM share from sciencemesh service + +https://github.com/cs3org/reva/pull/3695 +https://github.com/pondersource/sciencemesh-php/issues/166 \ No newline at end of file diff --git a/internal/http/services/sciencemesh/sciencemesh.go b/internal/http/services/sciencemesh/sciencemesh.go index e832583051..89f49990ee 100644 --- a/internal/http/services/sciencemesh/sciencemesh.go +++ b/internal/http/services/sciencemesh/sciencemesh.go @@ -95,6 +95,10 @@ func (s *svc) routerInit() error { if err := providersHandler.init(s.conf); err != nil { return err } + sharesHandler := new(sharesHandler) + if err := sharesHandler.init(s.conf); err != nil { + return err + } appsHandler := new(appsHandler) if err := appsHandler.init(s.conf); err != nil { @@ -106,8 +110,8 @@ func (s *svc) routerInit() error { s.router.Post("/accept-invite", tokenHandler.AcceptInvite) s.router.Get("/find-accepted-users", tokenHandler.FindAccepted) s.router.Get("/list-providers", providersHandler.ListProviders) + s.router.Post("/create-share", sharesHandler.CreateShare) s.router.Post("/open-in-app", appsHandler.OpenInApp) - return nil } diff --git a/internal/http/services/sciencemesh/share.go b/internal/http/services/sciencemesh/share.go new file mode 100644 index 0000000000..130cbd0505 --- /dev/null +++ b/internal/http/services/sciencemesh/share.go @@ -0,0 +1,176 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package sciencemesh + +import ( + "encoding/json" + "errors" + "mime" + "net/http" + + appprovider "github.com/cs3org/go-cs3apis/cs3/app/provider/v1beta1" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" + providerpb "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/internal/http/services/owncloud/ocs/conversions" + "github.com/cs3org/reva/internal/http/services/reqres" + "github.com/cs3org/reva/pkg/appctx" + "github.com/cs3org/reva/pkg/ocm/share" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/go-playground/validator/v10" +) + +var validate = validator.New() + +type sharesHandler struct { + gatewayClient gateway.GatewayAPIClient +} + +func (h *sharesHandler) init(c *config) error { + var err error + h.gatewayClient, err = pool.GetGatewayServiceClient(pool.Endpoint(c.GatewaySvc)) + return err +} + +type createShareRequest struct { + SourcePath string `json:"sourcePath" validate:"required"` + TargetPath string `json:"targetPath" validate:"required"` + Type string `json:"type"` + Role string `json:"role" validate:"oneof=viewer editor"` + RecipientUsername string `json:"recipientUsername" validate:"required"` + RecipientHost string `json:"recipientHost" validate:"required"` +} + +// CreateShare creates an OCM share. +func (h *sharesHandler) CreateShare(w http.ResponseWriter, r *http.Request) { + log := appctx.GetLogger(r.Context()) + + req, err := getCreateShareRequest(r) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorInvalidParameter, "invalid parameters", err) + return + } + + ctx := r.Context() + + statRes, err := h.gatewayClient.Stat(ctx, &providerpb.StatRequest{ + Ref: &providerpb.Reference{ + Path: req.SourcePath, + }, + }) + switch { + case err != nil: + reqres.WriteError(w, r, reqres.APIErrorServerError, "unexpected error", err) + return + case statRes.Status.Code == rpc.Code_CODE_NOT_FOUND: + reqres.WriteError(w, r, reqres.APIErrorNotFound, statRes.Status.Message, nil) + return + case statRes.Status.Code != rpc.Code_CODE_OK: + reqres.WriteError(w, r, reqres.APIErrorServerError, statRes.Status.Message, errors.New(statRes.Status.Message)) + return + } + + recipientProviderInfo, err := h.gatewayClient.GetInfoByDomain(ctx, &ocmprovider.GetInfoByDomainRequest{ + Domain: req.RecipientHost, + }) + if err != nil { + reqres.WriteError(w, r, reqres.APIErrorServerError, "error sending a grpc get invite by domain info request", err) + return + } + if recipientProviderInfo.Status.Code != rpc.Code_CODE_OK { + reqres.WriteError(w, r, reqres.APIErrorNotFound, recipientProviderInfo.Status.Message, errors.New(recipientProviderInfo.Status.Message)) + return + } + + perm, viewMode := getPermissionsByRole(req.Role) + + shareRes, err := h.gatewayClient.CreateOCMShare(ctx, &ocm.CreateOCMShareRequest{ + ResourceId: statRes.Info.Id, + Grantee: &providerpb.Grantee{ + Type: providerpb.GranteeType_GRANTEE_TYPE_USER, + Id: &providerpb.Grantee_UserId{ + UserId: &userpb.UserId{ + Idp: req.RecipientHost, + OpaqueId: req.RecipientUsername, + }, + }, + }, + RecipientMeshProvider: recipientProviderInfo.ProviderInfo, + AccessMethods: []*ocm.AccessMethod{ + share.NewWebDavAccessMethod(perm), + share.NewWebappAccessMethod(viewMode), + }, + }) + switch { + case err != nil: + reqres.WriteError(w, r, reqres.APIErrorServerError, "error sending a grpc CreateOCMShare", err) + return + case shareRes.Status.Code == rpc.Code_CODE_NOT_FOUND: + reqres.WriteError(w, r, reqres.APIErrorNotFound, shareRes.Status.Message, nil) + return + case shareRes.Status.Code == rpc.Code_CODE_ALREADY_EXISTS: + reqres.WriteError(w, r, reqres.APIErrorAlreadyExist, shareRes.Status.Message, nil) + return + case shareRes.Status.Code != rpc.Code_CODE_OK: + reqres.WriteError(w, r, reqres.APIErrorAlreadyExist, shareRes.Status.Message, errors.New(shareRes.Status.Message)) + return + } + + if err := json.NewEncoder(w).Encode(shareRes); err != nil { + log.Error().Err(err).Msg("error encoding response") + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) +} + +func getPermissionsByRole(role string) (*providerpb.ResourcePermissions, appprovider.ViewMode) { + switch role { + case "viewer": + return conversions.NewViewerRole().CS3ResourcePermissions(), appprovider.ViewMode_VIEW_MODE_READ_ONLY + case "editor": + return conversions.NewEditorRole().CS3ResourcePermissions(), appprovider.ViewMode_VIEW_MODE_READ_WRITE + } + return nil, 0 +} + +func getCreateShareRequest(r *http.Request) (*createShareRequest, error) { + var req createShareRequest + contentType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err == nil && contentType == "application/json" { + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return nil, err + } + } else { + return nil, errors.New("body request not recognised") + } + // set defaults + if req.Type == "" { + req.Type = "viewer" + } + // validate the request + if err := validate.Struct(req); err != nil { + return nil, err + } + return &req, nil +} From 80606f025e44841e332d55e086028e58c6405267 Mon Sep 17 00:00:00 2001 From: Antoon P Date: Thu, 4 May 2023 18:28:29 +0200 Subject: [PATCH 19/20] Data transfers new ocm impl (#3847) * * remove unnecessary creation of ocm reference in ocmshareprovider when accepting an ocm share * set transfer protocol when creating transfer type ocm share * refactor transfer endpoints for new ocm impl * refactor/cleanup gateway.ocmshareprovider.UpdateReceivedOCMShare() code * refactor data transfers folder config * new transfers config setting 'remove_on_cancel' * implement transfer destination path * update datatx example toml * update cli ocm-share-update-received with path flag * Add changelog * Add #PR --------- Co-authored-by: Antoon P --- changelog/unreleased/datatx-new-ocm-impl.md | 4 + cmd/reva/ocm-share-update-received.go | 28 ++ examples/datatx/datatx.toml | 86 ++-- internal/grpc/services/datatx/datatx.go | 25 +- internal/grpc/services/gateway/gateway.go | 4 - .../grpc/services/gateway/ocmshareprovider.go | 391 +++++++----------- .../ocmshareprovider/ocmshareprovider.go | 10 +- pkg/datatx/manager/rclone/rclone.go | 7 +- 8 files changed, 260 insertions(+), 295 deletions(-) create mode 100644 changelog/unreleased/datatx-new-ocm-impl.md diff --git a/changelog/unreleased/datatx-new-ocm-impl.md b/changelog/unreleased/datatx-new-ocm-impl.md new file mode 100644 index 0000000000..2ffad43d88 --- /dev/null +++ b/changelog/unreleased/datatx-new-ocm-impl.md @@ -0,0 +1,4 @@ +Enhancement: Update data transfers for current OCM shares implementation + +https://github.com/cs3org/reva/pull/3847 +https://github.com/cs3org/reva/issues/3846 \ No newline at end of file diff --git a/cmd/reva/ocm-share-update-received.go b/cmd/reva/ocm-share-update-received.go index 2006c721d3..92b5b2d80c 100644 --- a/cmd/reva/ocm-share-update-received.go +++ b/cmd/reva/ocm-share-update-received.go @@ -24,6 +24,7 @@ import ( rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" "github.com/pkg/errors" "google.golang.org/protobuf/types/known/fieldmaskpb" ) @@ -33,9 +34,11 @@ func ocmShareUpdateReceivedCommand() *command { cmd.Description = func() string { return "update a received OCM share" } cmd.Usage = func() string { return "Usage: ocm-share-update-received [-flags] " } state := cmd.String("state", "pending", "the state of the share (pending, accepted or rejected)") + path := cmd.String("path", "", "the destination path of the data transfer (ignored if this is not a transfer type share)") cmd.ResetFlags = func() { *state = "pending" + *path = "" } cmd.Action = func(w ...io.Writer) error { @@ -75,9 +78,25 @@ func ocmShareUpdateReceivedCommand() *command { } shareRes.Share.State = shareState + // check if we are dealing with a transfer in case the destination path needs to be set + _, ok := getTransferProtocol(shareRes.Share) + var opaque *typesv1beta1.Opaque + if ok { + // transfer_destination_path is not part of TransferProtocol and is specified as an opaque field + opaque = &typesv1beta1.Opaque{ + Map: map[string]*typesv1beta1.OpaqueEntry{ + "transfer_destination_path": { + Decoder: "plain", + Value: []byte(*path), + }, + }, + } + } + shareRequest := &ocm.UpdateReceivedOCMShareRequest{ Share: shareRes.Share, UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"state"}}, + Opaque: opaque, } updateRes, err := shareClient.UpdateReceivedOCMShare(ctx, shareRequest) @@ -95,6 +114,15 @@ func ocmShareUpdateReceivedCommand() *command { return cmd } +func getTransferProtocol(share *ocm.ReceivedShare) (*ocm.TransferProtocol, bool) { + for _, p := range share.Protocols { + if d, ok := p.Term.(*ocm.Protocol_TransferOptions); ok { + return d.TransferOptions, true + } + } + return nil, false +} + func getOCMShareState(state string) ocm.ShareState { switch state { case "pending": diff --git a/examples/datatx/datatx.toml b/examples/datatx/datatx.toml index f1271b3483..bb19821f9e 100644 --- a/examples/datatx/datatx.toml +++ b/examples/datatx/datatx.toml @@ -1,39 +1,47 @@ -# Example data transfer service configuration -[grpc.services.datatx] -# Rclone is the default data transfer driver -txdriver = "rclone" -# The shares,transfers db file (default: /var/tmp/reva/datatx-shares.json) -tx_shares_file = "" -# Base folder of the data transfers (default: /home/DataTransfers) -data_transfers_folder = "" - -# Rclone data transfer driver -[grpc.services.datatx.txdrivers.rclone] -# Rclone endpoint -endpoint = "http://..." -# Basic auth is used -auth_user = "...rcloneuser" -auth_pass = "...rcloneusersecret" -# The authentication scheme to use in the src and dest requests by rclone (follows the endpoints' authentication methods) -# Valid values: -# "bearer" (default) will result in rclone using request header: Authorization: "Bearer ...token..." -# "x-access-token" will result in rclone using request header: X-Access-Token: "...token..." -# If not set "bearer" is assumed -auth_header = "x-access-token" -# The transfers(jobs) db file (default: /var/tmp/reva/datatx-transfers.json) -file = "" -# Check status job interval in milliseconds -job_status_check_interval = 2000 -# The job timeout in milliseconds (must be long enough for big transfers!) -job_timeout = 120000 - -[http.services.ocdav] -# Rclone supports third-party copy push; for that to work with reva enable this setting -enable_http_tpc = true -# The authentication scheme reva uses for the tpc push call (the call to Destination). -# Follows the destination endpoint authentication method. -# Valid values: -# "bearer" (default) will result in header: Authorization: "Bearer ...token..." -# "x-access-token" will result in header: X-Access-Token: "...token..." -# If not set "bearer" is assumed -http_tpc_push_auth_header = "x-access-token" +# all relevant settings for data transfers + +[grpc.services.gateway] +datatx = "localhost:19000" +# base folder of the data transfers (eg. /home/DataTransfers) +data_transfers_folder = "" + + +[grpc.services.datatx] +# rclone is currently the only data transfer driver implementation +txdriver = "rclone" +# the shares,transfers db file (default: /var/tmp/reva/datatx-shares.json) +tx_shares_file = "" +# base folder of the data transfers (eg. /home/DataTransfers) +data_transfers_folder = "" + +# rclone driver +[grpc.services.datatx.txdrivers.rclone] +# rclone endpoint +endpoint = "http://..." +# Basic auth is used for authenticating with rclone +auth_user = "{rclone user}" +auth_pass = "{rclone user secret}" +# The authentication scheme to use in the src and dest requests by rclone (follows the endpoints' authentication methods) +# Valid values: +# "bearer" (default) will result in rclone using request header: Authorization: "Bearer ...token..." +# "x-access-token" will result in rclone using request header: X-Access-Token: "...token..." +# If not set "bearer" is assumed +auth_header = "x-access-token" +# the transfers(jobs) db file (default: /var/tmp/reva/datatx-transfers.json) +file = "" +# check status job interval in milliseconds +job_status_check_interval = 2000 +# the job timeout in milliseconds (must be long enough for big transfers!) +job_timeout = 120000 + +[http.services.ocdav] +# reva supports http third party copy +enable_http_tpc = true +# with rclone reva only supports http tpc push (ie. with the destination header specified) +# The authentication scheme reva uses for the tpc push call (the call to Destination). +# Follows the destination endpoint authentication method. +# Valid values: +# "bearer" (default) will result in header: Authorization: "Bearer ...token..." +# "x-access-token" will result in header: X-Access-Token: "...token..." +# If not set "bearer" is assumed +http_tpc_push_auth_header = "x-access-token" diff --git a/internal/grpc/services/datatx/datatx.go b/internal/grpc/services/datatx/datatx.go index c2a67aee6d..b3ac0b4398 100644 --- a/internal/grpc/services/datatx/datatx.go +++ b/internal/grpc/services/datatx/datatx.go @@ -46,10 +46,10 @@ type config struct { TxDriver string `mapstructure:"txdriver"` TxDrivers map[string]map[string]interface{} `mapstructure:"txdrivers"` // storage driver to persist share/transfer relation - StorageDriver string `mapstructure:"storage_driver"` - StorageDrivers map[string]map[string]interface{} `mapstructure:"storage_drivers"` - TxSharesFile string `mapstructure:"tx_shares_file"` - DataTransfersFolder string `mapstructure:"data_transfers_folder"` + StorageDriver string `mapstructure:"storage_driver"` + StorageDrivers map[string]map[string]interface{} `mapstructure:"storage_drivers"` + TxSharesFile string `mapstructure:"tx_shares_file"` + RemoveOnCancel bool `mapstructure:"remove_on_cancel"` } type service struct { @@ -81,9 +81,6 @@ func (c *config) init() { if c.TxSharesFile == "" { c.TxSharesFile = "/var/tmp/reva/datatx-shares.json" } - if c.DataTransfersFolder == "" { - c.DataTransfersFolder = "/home/DataTransfers" - } } func (s *service) Register(ss *grpc.Server) { @@ -211,10 +208,22 @@ func (s *service) CancelTransfer(ctx context.Context, req *datatx.CancelTransfer return nil, errtypes.InternalError("datatx service: transfer not found") } + transferRemovedMessage := "" + if s.conf.RemoveOnCancel { + delete(s.txShareDriver.model.TxShares, req.TxId.GetOpaqueId()) + if err := s.txShareDriver.model.saveTxShare(); err != nil { + err = errors.Wrap(err, "datatx service: error deleting transfer: "+datatx.Status_STATUS_INVALID.String()) + return &datatx.CancelTransferResponse{ + Status: status.NewInvalid(ctx, "error cancelling transfer"), + }, err + } + transferRemovedMessage = "transfer successfully removed" + } + txInfo, err := s.txManager.CancelTransfer(ctx, req.GetTxId().OpaqueId) if err != nil { txInfo.ShareId = &ocm.ShareId{OpaqueId: txShare.ShareID} - err = errors.Wrap(err, "datatx service: error cancelling transfer") + err = errors.Wrapf(err, "(%v) datatx service: error cancelling transfer", transferRemovedMessage) return &datatx.CancelTransferResponse{ Status: status.NewInternal(ctx, err, "error cancelling transfer"), TxInfo: txInfo, diff --git a/internal/grpc/services/gateway/gateway.go b/internal/grpc/services/gateway/gateway.go index 9e812aafe1..72776e5fb5 100644 --- a/internal/grpc/services/gateway/gateway.go +++ b/internal/grpc/services/gateway/gateway.go @@ -81,10 +81,6 @@ func (c *config) init() { c.ShareFolder = strings.Trim(c.ShareFolder, "/") - if c.DataTransfersFolder == "" { - c.DataTransfersFolder = "DataTransfers" - } - if c.TokenManager == "" { c.TokenManager = "jwt" } diff --git a/internal/grpc/services/gateway/ocmshareprovider.go b/internal/grpc/services/gateway/ocmshareprovider.go index 28019e786c..ddcadd44d3 100644 --- a/internal/grpc/services/gateway/ocmshareprovider.go +++ b/internal/grpc/services/gateway/ocmshareprovider.go @@ -25,7 +25,6 @@ import ( "path" "strings" - ocmprovider "github.com/cs3org/go-cs3apis/cs3/ocm/provider/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" ocm "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -171,6 +170,36 @@ func (s *svc) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateReceive }, nil } + // retrieve the persisted received share + getShareReq := &ocm.GetReceivedOCMShareRequest{ + Ref: &ocm.ShareReference{ + Spec: &ocm.ShareReference_Id{ + Id: req.Share.Id, + }, + }, + } + getShareRes, err := s.GetReceivedOCMShare(ctx, getShareReq) + if err != nil { + log.Err(err).Msg("gateway: error calling GetReceivedShare") + return &ocm.UpdateReceivedOCMShareResponse{ + Status: &rpc.Status{ + Code: rpc.Code_CODE_INTERNAL, + }, + }, nil + } + if getShareRes.Status.Code != rpc.Code_CODE_OK { + log.Error().Msg("gateway: error calling GetReceivedShare") + return &ocm.UpdateReceivedOCMShareResponse{ + Status: &rpc.Status{ + Code: rpc.Code_CODE_INTERNAL, + }, + }, nil + } + share := getShareRes.Share + if share == nil { + panic("gateway: error updating a received share: the share is nil") + } + res, err := c.UpdateReceivedOCMShare(ctx, req) if err != nil { log.Err(err).Msg("gateway: error calling UpdateReceivedShare") @@ -181,200 +210,16 @@ func (s *svc) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateReceive }, nil } - // properties are updated in the order they appear in the field mask - // when an error occurs the request ends and no further fields are updated for i := range req.UpdateMask.Paths { switch req.UpdateMask.Paths[i] { case "state": switch req.GetShare().GetState() { case ocm.ShareState_SHARE_STATE_ACCEPTED: - getShareReq := &ocm.GetReceivedOCMShareRequest{ - Ref: &ocm.ShareReference{ - Spec: &ocm.ShareReference_Id{ - Id: req.Share.Id, - }, - }, - } - getShareRes, err := s.GetReceivedOCMShare(ctx, getShareReq) - if err != nil { - log.Err(err).Msg("gateway: error calling GetReceivedShare") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{ - Code: rpc.Code_CODE_INTERNAL, - }, - }, nil - } - - if getShareRes.Status.Code != rpc.Code_CODE_OK { - log.Error().Msg("gateway: error calling GetReceivedShare") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{ - Code: rpc.Code_CODE_INTERNAL, - }, - }, nil - } - - share := getShareRes.Share - if share == nil { - panic("gateway: error updating a received share: the share is nil") - } - - if isTransferShare(share) { - srcIdp := share.GetOwner().GetIdp() - meshProvider, err := s.GetInfoByDomain(ctx, &ocmprovider.GetInfoByDomainRequest{ - Domain: srcIdp, - }) - if err != nil { - log.Err(err).Msg("gateway: error calling GetInfoByDomain") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{Code: rpc.Code_CODE_INTERNAL}, - }, nil - } - var srcServiceHost string - var srcEndpointPath string - // target URI scheme will be the webdav endpoint scheme - var srcEndpointScheme string - for _, s := range meshProvider.ProviderInfo.Services { - if strings.ToLower(s.Endpoint.Type.Name) == "webdav" { - srcWebdavEndpointURL, err := url.Parse(s.Endpoint.Path) - if err != nil || srcWebdavEndpointURL.Host == "" { - log.Err(err).Msg("gateway: error calling UpdateReceivedShare: unable to parse webdav endpoint \"" + s.Endpoint.Path + "\" into URL structure") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{Code: rpc.Code_CODE_INTERNAL}, - }, nil - } - var srcWebdavHostURLString string - if strings.Contains(s.Host, "://") { - srcWebdavHostURLString = s.Host - } else { - srcWebdavHostURLString = "http://" + s.Host - } - srcWebdavHostURL, err := url.Parse(srcWebdavHostURLString) - if err != nil { - log.Err(err).Msg("gateway: error calling UpdateReceivedShare: unable to parse webdav service host \"" + s.Host + "\" into URL structure") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{Code: rpc.Code_CODE_INTERNAL}, - }, nil - } - srcServiceHost = srcWebdavHostURL.Host + srcWebdavHostURL.Path - // optional prefix must only appear in target url path: - // http://...token...@reva.eu/prefix/?name=remote.php/webdav/home/... - srcEndpointPath = strings.TrimPrefix(srcWebdavEndpointURL.Path, srcWebdavHostURL.Path) - srcEndpointScheme = srcWebdavEndpointURL.Scheme - break - } - } - - var srcToken string - srcTokenOpaque, ok := share.Grantee.Opaque.Map["token"] - if !ok { - return &ocm.UpdateReceivedOCMShareResponse{ - Status: status.NewNotFound(ctx, "token not found"), - }, nil - } - switch srcTokenOpaque.Decoder { - case "plain": - srcToken = string(srcTokenOpaque.Value) - default: - err := errtypes.NotSupported("opaque entry decoder not recognized: " + srcTokenOpaque.Decoder) - return &ocm.UpdateReceivedOCMShareResponse{ - Status: status.NewInternal(ctx, err, "error updating received share"), - }, nil - } - - srcPath := path.Join(srcEndpointPath, share.Name) - srcTargetURI := fmt.Sprintf("%s://%s@%s?name=%s", srcEndpointScheme, srcToken, srcServiceHost, srcPath) - - // get the webdav endpoint of the grantee's idp - var granteeIdp string - if share.GetGrantee().Type == provider.GranteeType_GRANTEE_TYPE_USER { - granteeIdp = share.GetGrantee().GetUserId().Idp - } - if share.GetGrantee().Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - granteeIdp = share.GetGrantee().GetGroupId().Idp - } - destWebdavEndpoint, err := s.getWebdavEndpoint(ctx, granteeIdp) - if err != nil { - log.Err(err).Msg("gateway: error calling UpdateReceivedShare") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{Code: rpc.Code_CODE_INTERNAL}, - }, nil - } - destWebdavEndpointURL, err := url.Parse(destWebdavEndpoint) - if err != nil { - log.Err(err).Msg("gateway: error calling UpdateReceivedShare: unable to parse webdav endpoint \"" + destWebdavEndpoint + "\" into URL structure") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{Code: rpc.Code_CODE_INTERNAL}, - }, nil - } - destWebdavHost, err := s.getWebdavHost(ctx, granteeIdp) - if err != nil { - log.Err(err).Msg("gateway: error calling UpdateReceivedShare") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{Code: rpc.Code_CODE_INTERNAL}, - }, nil - } - var dstWebdavURLString string - if strings.Contains(destWebdavHost, "://") { - dstWebdavURLString = destWebdavHost - } else { - dstWebdavURLString = "http://" + destWebdavHost - } - dstWebdavHostURL, err := url.Parse(dstWebdavURLString) - if err != nil { - log.Err(err).Msg("gateway: error calling UpdateReceivedShare: unable to parse webdav service host \"" + dstWebdavURLString + "\" into URL structure") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{Code: rpc.Code_CODE_INTERNAL}, - }, nil - } - destServiceHost := dstWebdavHostURL.Host + dstWebdavHostURL.Path - // optional prefix must only appear in target url path: - // http://...token...@reva.eu/prefix/?name=remote.php/webdav/home/... - destEndpointPath := strings.TrimPrefix(destWebdavEndpointURL.Path, dstWebdavHostURL.Path) - destEndpointScheme := destWebdavEndpointURL.Scheme - destToken := ctxpkg.ContextMustGetToken(ctx) - homeRes, err := s.GetHome(ctx, &provider.GetHomeRequest{}) - if err != nil { - log.Err(err).Msg("gateway: error calling UpdateReceivedShare") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{Code: rpc.Code_CODE_INTERNAL}, - }, nil - } - destPath := path.Join(destEndpointPath, homeRes.Path, s.c.DataTransfersFolder, path.Base(share.Name)) - destTargetURI := fmt.Sprintf("%s://%s@%s?name=%s", destEndpointScheme, destToken, destServiceHost, destPath) - - shareID := &ocm.ShareId{ - OpaqueId: share.GetId().OpaqueId, - } - req := &datatx.CreateTransferRequest{ - SrcTargetUri: srcTargetURI, - DestTargetUri: destTargetURI, - ShareId: shareID, - } - res, err := s.CreateTransfer(ctx, req) - if err != nil { - log.Err(err).Msg("gateway: error calling CreateTransfer") - return &ocm.UpdateReceivedOCMShareResponse{ - Status: &rpc.Status{ - Code: rpc.Code_CODE_INTERNAL, - }, - }, err - } - - log.Info().Msgf("gateway: CreateTransfer: %v", res.TxInfo) - - // do not create an OCM reference, just return - return &ocm.UpdateReceivedOCMShareResponse{ - Status: status.NewOK(ctx), - }, nil - } - - createRefStatus, err := s.createOCMReference(ctx, share) - return &ocm.UpdateReceivedOCMShareResponse{ - Status: createRefStatus, - }, err + // for a transfer this is handled elsewhere + case ocm.ShareState_SHARE_STATE_PENDING: + // currently no consequences case ocm.ShareState_SHARE_STATE_REJECTED: - s.removeReference(ctx, req.GetShare().ResourceId) // error is logged inside removeReference + s.removeReference(ctx, share.ResourceId) // error is logged inside removeReference // FIXME we are ignoring an error from removeReference here return res, nil } @@ -388,14 +233,126 @@ func (s *svc) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateReceive return nil, errtypes.NotSupported("updating " + req.UpdateMask.Paths[i] + " is not supported") } } + // handle transfer in case it has not already been accepted + if s.isTransferShare(share) && req.GetShare().State == ocm.ShareState_SHARE_STATE_ACCEPTED && share.State != ocm.ShareState_SHARE_STATE_ACCEPTED { + // get provided destination path + transferDestinationPath, err := s.getTransferDestinationPath(ctx, req) + if err != nil { + if err != nil { + log.Err(err).Msg("gateway: error calling UpdateReceivedShare") + return &ocm.UpdateReceivedOCMShareResponse{ + Status: &rpc.Status{ + Code: rpc.Code_CODE_INTERNAL, + }, + }, err + } + } + + error := s.handleTransfer(ctx, share, transferDestinationPath) + if error != nil { + log.Err(error).Msg("gateway: error handling transfer in UpdateReceivedShare") + return &ocm.UpdateReceivedOCMShareResponse{ + Status: &rpc.Status{ + Code: rpc.Code_CODE_INTERNAL, + }, + }, error + } + } return res, nil } -func isTransferShare(s *ocm.ReceivedShare) bool { - _, ok := getTransferProtocol(s) +func (s *svc) handleTransfer(ctx context.Context, share *ocm.ReceivedShare, transferDestinationPath string) error { + log := appctx.GetLogger(ctx) + + protocol, ok := s.getTransferProtocol(share) + if !ok { + return errors.New("gateway: unable to retrieve transfer protocol") + } + sourceURI := protocol.SourceUri + + // get the webdav endpoint of the grantee's idp + var granteeIdp string + if share.GetGrantee().Type == provider.GranteeType_GRANTEE_TYPE_USER { + granteeIdp = share.GetGrantee().GetUserId().Idp + } + if share.GetGrantee().Type == provider.GranteeType_GRANTEE_TYPE_GROUP { + granteeIdp = share.GetGrantee().GetGroupId().Idp + } + destWebdavEndpoint, err := s.getWebdavEndpoint(ctx, granteeIdp) + if err != nil { + log.Err(err).Msg("gateway: error calling UpdateReceivedShare") + return err + } + destWebdavEndpointURL, err := url.Parse(destWebdavEndpoint) + if err != nil { + log.Err(err).Msg("gateway: error calling UpdateReceivedShare: unable to parse webdav endpoint \"" + destWebdavEndpoint + "\" into URL structure") + return err + } + destWebdavHost, err := s.getWebdavHost(ctx, granteeIdp) + if err != nil { + log.Err(err).Msg("gateway: error calling UpdateReceivedShare") + return err + } + var dstWebdavURLString string + if strings.Contains(destWebdavHost, "://") { + dstWebdavURLString = destWebdavHost + } else { + dstWebdavURLString = "http://" + destWebdavHost + } + dstWebdavHostURL, err := url.Parse(dstWebdavURLString) + if err != nil { + log.Err(err).Msg("gateway: error calling UpdateReceivedShare: unable to parse webdav service host \"" + dstWebdavURLString + "\" into URL structure") + return err + } + destServiceHost := dstWebdavHostURL.Host + dstWebdavHostURL.Path + // optional prefix must only appear in target url path: + // http://...token...@reva.eu/prefix/?name=remote.php/webdav/home/... + destEndpointPath := strings.TrimPrefix(destWebdavEndpointURL.Path, dstWebdavHostURL.Path) + destEndpointScheme := destWebdavEndpointURL.Scheme + destToken := ctxpkg.ContextMustGetToken(ctx) + destPath := path.Join(destEndpointPath, transferDestinationPath, path.Base(share.Name)) + destTargetURI := fmt.Sprintf("%s://%s@%s?name=%s", destEndpointScheme, destToken, destServiceHost, destPath) + // var destUri string + req := &datatx.CreateTransferRequest{ + SrcTargetUri: sourceURI, + DestTargetUri: destTargetURI, + ShareId: share.Id, + } + + res, err := s.CreateTransfer(ctx, req) + if err != nil { + return err + } + log.Info().Msgf("gateway: CreateTransfer: %v", res.TxInfo) + return nil +} + +func (s *svc) isTransferShare(share *ocm.ReceivedShare) bool { + _, ok := s.getTransferProtocol(share) return ok } +func (s *svc) getTransferDestinationPath(ctx context.Context, req *ocm.UpdateReceivedOCMShareRequest) (string, error) { + log := appctx.GetLogger(ctx) + // the destination path is not part of any protocol, but an opaque field + destPathOpaque, ok := req.GetOpaque().GetMap()["transfer_destination_path"] + if ok { + switch destPathOpaque.Decoder { + case "plain": + if string(destPathOpaque.Value) != "" { + return string(destPathOpaque.Value), nil + } + default: + return "", errtypes.NotSupported("decoder of opaque entry 'transfer_destination_path' not recognized: " + destPathOpaque.Decoder) + } + } + log.Info().Msg("destination path not provided, trying default transfer destination folder") + if s.c.DataTransfersFolder == "" { + return "", errtypes.NotSupported("no destination path provided and default transfer destination folder is not set") + } + return s.c.DataTransfersFolder, nil +} + func (s *svc) GetReceivedOCMShare(ctx context.Context, req *ocm.GetReceivedOCMShareRequest) (*ocm.GetReceivedOCMShareResponse, error) { c, err := pool.GetOCMShareProviderClient(pool.Endpoint(s.c.OCMShareProviderEndpoint)) if err != nil { @@ -413,7 +370,7 @@ func (s *svc) GetReceivedOCMShare(ctx context.Context, req *ocm.GetReceivedOCMSh return res, nil } -func getTransferProtocol(share *ocm.ReceivedShare) (*ocm.TransferProtocol, bool) { +func (s *svc) getTransferProtocol(share *ocm.ReceivedShare) (*ocm.TransferProtocol, bool) { for _, p := range share.Protocols { if d, ok := p.Term.(*ocm.Protocol_TransferOptions); ok { return d.TransferOptions, true @@ -421,53 +378,3 @@ func getTransferProtocol(share *ocm.ReceivedShare) (*ocm.TransferProtocol, bool) } return nil, false } - -func (s *svc) createOCMReference(ctx context.Context, share *ocm.ReceivedShare) (*rpc.Status, error) { - log := appctx.GetLogger(ctx) - - d, _ := getTransferProtocol(share) - - homeRes, err := s.GetHome(ctx, &provider.GetHomeRequest{}) - if err != nil { - err := errors.Wrap(err, "gateway: error calling GetHome") - return status.NewInternal(ctx, err, "error updating received share"), nil - } - - var refPath, targetURI string - // reference path is the home path + some name on the corresponding - // mesh provider (/home/MyShares/x) - // It is the responsibility of the gateway to resolve these references and merge the response back - // from the main request. - refPath = path.Join(homeRes.Path, s.c.ShareFolder, path.Base(share.Name)) - // webdav is the scheme, token@host the opaque part and the share name the query of the URL. - targetURI = fmt.Sprintf("webdav://%s@%s?name=%s", d.SharedSecret, share.Creator.Idp, share.Name) - - log.Info().Msg("mount path will be:" + refPath) - createRefReq := &provider.CreateReferenceRequest{ - Ref: &provider.Reference{Path: refPath}, - TargetUri: targetURI, - } - - c, err := s.findByPath(ctx, refPath) - if err != nil { - if _, ok := err.(errtypes.IsNotFound); ok { - return status.NewNotFound(ctx, "storage provider not found"), nil - } - return status.NewInternal(ctx, err, "error finding storage provider"), nil - } - - createRefRes, err := c.CreateReference(ctx, createRefReq) - if err != nil { - log.Err(err).Msg("gateway: error calling GetHome") - return &rpc.Status{ - Code: rpc.Code_CODE_INTERNAL, - }, nil - } - - if createRefRes.Status.Code != rpc.Code_CODE_OK { - err := status.NewErrorFromCode(createRefRes.Status.GetCode(), "gateway") - return status.NewInternal(ctx, err, "error updating received share"), nil - } - - return status.NewOK(ctx), nil -} diff --git a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go index 87401ccecb..9112af18fe 100644 --- a/internal/grpc/services/ocmshareprovider/ocmshareprovider.go +++ b/internal/grpc/services/ocmshareprovider/ocmshareprovider.go @@ -208,6 +208,14 @@ func (s *service) getWebappProtocol(share *ocm.Share) *ocmd.Webapp { } } +func (s *service) getDataTransferProtocol(ctx context.Context, share *ocm.Share) *ocmd.Datatx { + // TODO discover the size + return &ocmd.Datatx{ + SourceURI: s.webdavURL(ctx, share), + Size: 0, + } +} + func (s *service) getProtocols(ctx context.Context, share *ocm.Share) ocmd.Protocols { var p ocmd.Protocols for _, m := range share.AccessMethods { @@ -217,7 +225,7 @@ func (s *service) getProtocols(ctx context.Context, share *ocm.Share) ocmd.Proto case *ocm.AccessMethod_WebappOptions: p = append(p, s.getWebappProtocol(share)) case *ocm.AccessMethod_TransferOptions: - // TODO + p = append(p, s.getDataTransferProtocol(ctx, share)) } } return p diff --git a/pkg/datatx/manager/rclone/rclone.go b/pkg/datatx/manager/rclone/rclone.go index 25df9aaeef..21e3d6e3f0 100644 --- a/pkg/datatx/manager/rclone/rclone.go +++ b/pkg/datatx/manager/rclone/rclone.go @@ -879,8 +879,13 @@ func (driver *rclone) extractEndpointInfo(ctx context.Context, targetURL string) return nil, errors.Wrap(err, "datatx service: error parsing target resource name") } + var path string + if m["name"] != nil { + path = m["name"][0] + } + return &endpoint{ - filePath: m["name"][0], + filePath: path, endpoint: uri.Host + uri.Path, endpointScheme: uri.Scheme, token: uri.User.String(), From 1c681a3509d4b05933fe8c4f1f283ce4e195a284 Mon Sep 17 00:00:00 2001 From: Javier Ferrer Date: Fri, 5 May 2023 10:52:17 +0200 Subject: [PATCH 20/20] Serverless services (#3824) * Add serverless services * Load serverless services * Start serverless services on launch * Codacy changes * Changelog * Example serverless config * Add example serverless service * Add service name to logger Co-authored-by: Gianmaria Del Monte * Simplify function call Co-authored-by: Gianmaria Del Monte * Exit with errors if initserverless fails Co-authored-by: Gianmaria Del Monte * Add signal handling to serverless services * Use context to pass timeout on service stop --------- Co-authored-by: Gianmaria Del Monte --- changelog/unreleased/serverless-services.md | 7 + cmd/revad/internal/grace/grace.go | 26 +++ cmd/revad/runtime/loader.go | 1 + cmd/revad/runtime/runtime.go | 48 +++++- .../serverless-example/notifications.toml | 32 ++++ .../services/helloworld/helloworld.go | 98 +++++++++++ internal/serverless/services/loader/loader.go | 25 +++ pkg/rhttp/rhttp.go | 3 +- pkg/rserverless/rserverless.go | 161 ++++++++++++++++++ 9 files changed, 394 insertions(+), 7 deletions(-) create mode 100644 changelog/unreleased/serverless-services.md create mode 100644 examples/serverless-example/notifications.toml create mode 100644 internal/serverless/services/helloworld/helloworld.go create mode 100644 internal/serverless/services/loader/loader.go create mode 100644 pkg/rserverless/rserverless.go diff --git a/changelog/unreleased/serverless-services.md b/changelog/unreleased/serverless-services.md new file mode 100644 index 0000000000..054999b47c --- /dev/null +++ b/changelog/unreleased/serverless-services.md @@ -0,0 +1,7 @@ +Enhancement: Serverless Services + +New type of service (along with http and grpc) +which does not have a listening server. Useful for +the notifications service and others in the future. + +https://github.com/cs3org/reva/pull/3824 diff --git a/cmd/revad/internal/grace/grace.go b/cmd/revad/internal/grace/grace.go index 16f43165a7..52cd3619a8 100644 --- a/cmd/revad/internal/grace/grace.go +++ b/cmd/revad/internal/grace/grace.go @@ -41,6 +41,7 @@ type Watcher struct { ppid int lns map[string]net.Listener ss map[string]Server + SL Serverless pidFile string childPIDs []int } @@ -254,6 +255,12 @@ type Server interface { Address() string } +// Serverless is the interface that the serverless server implements. +type Serverless interface { + Stop() error + GracefulStop() error +} + // TrapSignals captures the OS signal. func (w *Watcher) TrapSignals() { signalCh := make(chan os.Signal, 1024) @@ -293,6 +300,11 @@ func (w *Watcher) TrapSignals() { } w.log.Info().Msgf("fd to %s:%s abruptly closed", s.Network(), s.Address()) } + err := w.SL.Stop() + if err != nil { + w.log.Error().Err(err).Msg("error stopping serverless server") + } + w.log.Info().Msg("serverless services abruptly closed") w.Exit(1) } } @@ -306,6 +318,14 @@ func (w *Watcher) TrapSignals() { w.Exit(1) } } + if w.SL != nil { + err := w.SL.GracefulStop() + if err != nil { + w.log.Error().Err(err).Msg("error stopping server") + w.log.Info().Msg("exit with error code 1") + w.Exit(1) + } + } w.log.Info().Msg("exit with error code 0") w.Exit(0) case syscall.SIGINT, syscall.SIGTERM: @@ -317,6 +337,12 @@ func (w *Watcher) TrapSignals() { w.log.Error().Err(err).Msg("error stopping server") } } + err := w.SL.Stop() + if err != nil { + w.log.Error().Err(err).Msg("error stopping serverless server") + } + w.log.Info().Msg("serverless services abruptly closed") + w.Exit(0) } } diff --git a/cmd/revad/runtime/loader.go b/cmd/revad/runtime/loader.go index da63b369ce..52488936d3 100644 --- a/cmd/revad/runtime/loader.go +++ b/cmd/revad/runtime/loader.go @@ -27,6 +27,7 @@ import ( _ "github.com/cs3org/reva/internal/http/interceptors/auth/tokenwriter/loader" _ "github.com/cs3org/reva/internal/http/interceptors/loader" _ "github.com/cs3org/reva/internal/http/services/loader" + _ "github.com/cs3org/reva/internal/serverless/services/loader" _ "github.com/cs3org/reva/pkg/app/provider/loader" _ "github.com/cs3org/reva/pkg/app/registry/loader" _ "github.com/cs3org/reva/pkg/appauth/manager/loader" diff --git a/cmd/revad/runtime/runtime.go b/cmd/revad/runtime/runtime.go index c2b64aa001..255c55bdf8 100644 --- a/cmd/revad/runtime/runtime.go +++ b/cmd/revad/runtime/runtime.go @@ -33,6 +33,7 @@ import ( "github.com/cs3org/reva/pkg/registry/memory" "github.com/cs3org/reva/pkg/rgrpc" "github.com/cs3org/reva/pkg/rhttp" + "github.com/cs3org/reva/pkg/rserverless" "github.com/cs3org/reva/pkg/sharedconf" rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" @@ -93,13 +94,23 @@ func run(mainConf map[string]interface{}, coreConf *coreConf, logger *zerolog.Lo initCPUCount(coreConf, logger) servers := initServers(mainConf, logger) + serverless := initServerless(mainConf, logger) + + if len(servers) == 0 && serverless == nil { + logger.Info().Msg("nothing to do, no grpc/http/serverless enabled_services declared in config") + os.Exit(1) + } + watcher, err := initWatcher(logger, filename) if err != nil { log.Panic(err) } listeners := initListeners(watcher, servers, logger) + if serverless != nil { + watcher.SL = serverless + } - start(mainConf, servers, listeners, logger, watcher) + start(mainConf, servers, serverless, listeners, logger, watcher) } func initListeners(watcher *grace.Watcher, servers map[string]grace.Server, log *zerolog.Logger) map[string]net.Listener { @@ -141,13 +152,22 @@ func initServers(mainConf map[string]interface{}, log *zerolog.Logger) map[strin servers["grpc"] = s } - if len(servers) == 0 { - log.Info().Msg("nothing to do, no grpc/http enabled_services declared in config") - os.Exit(1) - } return servers } +func initServerless(mainConf map[string]interface{}, log *zerolog.Logger) *rserverless.Serverless { + if isEnabledServerless(mainConf) { + serverless, err := getServerless(mainConf["serverless"], log) + if err != nil { + log.Error().Err(err).Msg("error") + os.Exit(1) + } + return serverless + } + + return nil +} + func initTracing(conf *coreConf) { rtrace.SetTraceProvider(conf.TracingCollector, conf.TracingEndpoint, conf.TracingServiceName) } @@ -184,7 +204,7 @@ func handlePIDFlag(l *zerolog.Logger, pidFile string) (*grace.Watcher, error) { return w, nil } -func start(mainConf map[string]interface{}, servers map[string]grace.Server, listeners map[string]net.Listener, log *zerolog.Logger, watcher *grace.Watcher) { +func start(mainConf map[string]interface{}, servers map[string]grace.Server, serverless *rserverless.Serverless, listeners map[string]net.Listener, log *zerolog.Logger, watcher *grace.Watcher) { if isEnabledHTTP(mainConf) { go func() { if err := servers["http"].(*rhttp.Server).Start(listeners["http"]); err != nil { @@ -201,6 +221,13 @@ func start(mainConf map[string]interface{}, servers map[string]grace.Server, lis } }() } + if isEnabledServerless(mainConf) { + if err := serverless.Start(); err != nil { + log.Error().Err(err).Msg("error starting serverless services") + watcher.Exit(1) + } + } + watcher.TrapSignals() } @@ -264,6 +291,11 @@ func getHTTPServer(conf interface{}, l *zerolog.Logger) (*rhttp.Server, error) { return s, nil } +func getServerless(conf interface{}, l *zerolog.Logger) (*rserverless.Serverless, error) { + sub := l.With().Str("pkg", "rserverless").Logger() + return rserverless.New(conf, sub) +} + // adjustCPU parses string cpu and sets GOMAXPROCS // // according to its value. It accepts either @@ -365,6 +397,10 @@ func isEnabledGRPC(conf map[string]interface{}) bool { return isEnabled("grpc", conf) } +func isEnabledServerless(conf map[string]interface{}) bool { + return isEnabled("serverless", conf) +} + func isEnabled(key string, conf map[string]interface{}) bool { if a, ok := conf[key]; ok { if b, ok := a.(map[string]interface{}); ok { diff --git a/examples/serverless-example/notifications.toml b/examples/serverless-example/notifications.toml new file mode 100644 index 0000000000..f1fee178f2 --- /dev/null +++ b/examples/serverless-example/notifications.toml @@ -0,0 +1,32 @@ +[log] +output = "/var/log/revad/revad-notifications.log" +mode = "json" + +[shared] +gatewaysvc = "localhost:19000" +jwt_secret = "Pive-Fumkiu4" +skip_user_groups_in_token = true + +[serverless.services.notifications] +nats_address = "nats-server-01.example.com" +nats_token = "secret-token-example" +nats_template_subject = "reva-notifications-template" +nats_notification_subject = "reva-notifications-notification" +nats_trigger_subject = "reva-notifications-trigger" +storage_driver = "sql" +grouping_interval = 60 +grouping_maxsize = 100 + +[serverless.services.notifications.storage_drivers.sql] +db_username = "username" +db_password = "password" +db_host = "database.example.com" +db_port = 3306 +db_name = "notifications" + +[serverless.services.notifications.handlers.email] +smtp_server = "mx.example.com:25" +disable_auth = true +default_sender = "noreply@cernbox.cern.ch" + +[tracing] diff --git a/internal/serverless/services/helloworld/helloworld.go b/internal/serverless/services/helloworld/helloworld.go new file mode 100644 index 0000000000..50120cda11 --- /dev/null +++ b/internal/serverless/services/helloworld/helloworld.go @@ -0,0 +1,98 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package helloworld + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/cs3org/reva/pkg/rserverless" + "github.com/mitchellh/mapstructure" + "github.com/rs/zerolog" +) + +type config struct { + Outfile string `mapstructure:"outfile"` +} + +func (c *config) init() { + if c.Outfile == "" { + c.Outfile = "/tmp/revad-helloworld-hello" + } +} + +type svc struct { + conf *config + file *os.File + log *zerolog.Logger +} + +func init() { + rserverless.Register("helloworld", New) +} + +// New returns a new helloworld service. +func New(m map[string]interface{}, log *zerolog.Logger) (rserverless.Service, error) { + conf := &config{} + conf.init() + + if err := mapstructure.Decode(m, conf); err != nil { + return nil, err + } + + file, err := os.OpenFile(conf.Outfile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Err(err) + return nil, err + } + + s := &svc{ + conf: conf, + log: log, + file: file, + } + + return s, nil +} + +// Start starts the helloworld service. +func (s *svc) Start() { + s.log.Debug().Msgf("helloworld server started, saying hello at %s", s.conf.Outfile) + go s.sayHello(s.conf.Outfile) +} + +// Close stops the helloworld service. +func (s *svc) Close(ctx context.Context) error { + return s.file.Close() +} + +func (s *svc) sayHello(filename string) { + for { + s.log.Info().Msg("saying hello") + h := fmt.Sprintf("%s - hello world!\n", time.Now().String()) + + _, err := s.file.Write([]byte(h)) + if err != nil { + s.log.Err(err) + } + time.Sleep(5 * time.Second) + } +} diff --git a/internal/serverless/services/loader/loader.go b/internal/serverless/services/loader/loader.go new file mode 100644 index 0000000000..1b466a144c --- /dev/null +++ b/internal/serverless/services/loader/loader.go @@ -0,0 +1,25 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package loader + +import ( + // Load core serverless services. + _ "github.com/cs3org/reva/internal/serverless/services/helloworld" + // Add your own service here. +) diff --git a/pkg/rhttp/rhttp.go b/pkg/rhttp/rhttp.go index 9bd689f405..de000aaac1 100644 --- a/pkg/rhttp/rhttp.go +++ b/pkg/rhttp/rhttp.go @@ -200,7 +200,8 @@ func (s *Server) registerServices() error { for svcName := range s.conf.Services { if s.isServiceEnabled(svcName) { newFunc := global.Services[svcName] - svc, err := newFunc(s.conf.Services[svcName], &s.log) + svcLogger := s.log.With().Str("service", svcName).Logger() + svc, err := newFunc(s.conf.Services[svcName], &svcLogger) if err != nil { err = errors.Wrapf(err, "http service %s could not be started,", svcName) return err diff --git a/pkg/rserverless/rserverless.go b/pkg/rserverless/rserverless.go new file mode 100644 index 0000000000..7af26640c5 --- /dev/null +++ b/pkg/rserverless/rserverless.go @@ -0,0 +1,161 @@ +// Copyright 2018-2023 CERN +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// In applying this license, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +package rserverless + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" + "github.com/rs/zerolog" +) + +// Service represents a serverless service. +type Service interface { + Start() + Close(ctx context.Context) error +} + +// Services is a map of service name and its new function. +var Services = map[string]NewService{} + +// Register registers a new serverless service with name and new function. +func Register(name string, newFunc NewService) { + Services[name] = newFunc +} + +// NewService is the function that serverless services need to register at init time. +type NewService func(conf map[string]interface{}, log *zerolog.Logger) (Service, error) + +// Serverless contains the serveless collection of services. +type Serverless struct { + conf *config + log zerolog.Logger + services map[string]Service +} + +type config struct { + Services map[string]map[string]interface{} `mapstructure:"services"` +} + +// New returns a new serverless collection of services. +func New(m interface{}, l zerolog.Logger) (*Serverless, error) { + conf := &config{} + if err := mapstructure.Decode(m, conf); err != nil { + return nil, err + } + + n := &Serverless{ + conf: conf, + log: l, + services: map[string]Service{}, + } + return n, nil +} + +func (s *Serverless) isServiceEnabled(svcName string) bool { + _, ok := Services[svcName] + return ok +} + +// Start starts the serverless service collection. +func (s *Serverless) Start() error { + return s.registerAndStartServices() +} + +// GracefulStop gracefully stops the serverless services. +func (s *Serverless) GracefulStop() error { + var wg sync.WaitGroup + + for svcName, svc := range s.services { + wg.Add(1) + + go func(svcName string, svc Service) { + defer wg.Done() + + s.log.Info().Msgf("Sending stop request to service %s", svcName) + ctx := context.Background() + + err := svc.Close(ctx) + if err != nil { + s.log.Error().Err(err).Msgf("error stopping service %s", svcName) + } else { + s.log.Info().Msgf("service %s stopped", svcName) + } + }(svcName, svc) + } + + wg.Wait() + + return nil +} + +// Stop stops the serverless services with a one second deadline. +func (s *Serverless) Stop() error { + var wg sync.WaitGroup + + for svcName, svc := range s.services { + wg.Add(1) + + go func(svcName string, svc Service) { + defer wg.Done() + + s.log.Info().Msgf("Sending stop request to service %s", svcName) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + err := svc.Close(ctx) + if err != nil { + s.log.Error().Err(err).Msgf("error stopping service %s", svcName) + } else { + s.log.Info().Msgf("service %s stopped", svcName) + } + }(svcName, svc) + } + + wg.Wait() + + return nil +} + +func (s *Serverless) registerAndStartServices() error { + for svcName := range s.conf.Services { + if s.isServiceEnabled(svcName) { + newFunc := Services[svcName] + svcLogger := s.log.With().Str("service", svcName).Logger() + svc, err := newFunc(s.conf.Services[svcName], &svcLogger) + if err != nil { + return errors.Wrapf(err, "serverless service %s could not be initialized", svcName) + } + + go svc.Start() + + s.services[svcName] = svc + + s.log.Info().Msgf("serverless service enabled: %s", svcName) + } else { + return fmt.Errorf("serverless service %s does not exist", svcName) + } + } + + return nil +}