diff --git a/changelog/unreleased/wrap-unwrap-in-gateway.md b/changelog/unreleased/wrap-unwrap-in-gateway.md new file mode 100644 index 0000000000..bce2c6263f --- /dev/null +++ b/changelog/unreleased/wrap-unwrap-in-gateway.md @@ -0,0 +1,5 @@ +Change: move wrapping and unwrapping of paths to the storage gateway + +We've moved the wrapping and unwrapping of reference paths to the storage gateway so that the storageprovider doesn't have to know its mount path. + +https://github.com/cs3org/reva/pull/2016 diff --git a/internal/grpc/interceptors/auth/scope.go b/internal/grpc/interceptors/auth/scope.go index 113a67a311..195145911f 100644 --- a/internal/grpc/interceptors/auth/scope.go +++ b/internal/grpc/interceptors/auth/scope.go @@ -203,6 +203,26 @@ func checkIfNestedResource(ctx context.Context, ref *provider.Reference, parent return strings.HasPrefix(childPath, parentPath), nil + // resourcePath := statResponse.Info.Path + + // if strings.HasPrefix(ref.GetPath(), resourcePath) { + // // The path corresponds to the resource to which the token has access. + // // We allow access to it. + // return true, nil + // } + + // // If we arrived here that could mean that ref.GetPath is not prefixed with the storage mount path but resourcePath is + // // because it was returned by the gateway which will prefix it. To fix that we remove the mount path from the resourcePath. + // // resourcePath = "/users//some/path" + // // After the split we have [" ", "users", "/some/path"]. + // trimmedPath := "/" + strings.SplitN(resourcePath, "/", 3)[2] + // if strings.HasPrefix(ref.GetPath(), trimmedPath) { + // // The path corresponds to the resource to which the token has access. + // // We allow access to it. + // return true, nil + // } + + // return false, nil } func extractRef(req interface{}) (*provider.Reference, bool) { diff --git a/internal/grpc/services/gateway/ocmshareprovider.go b/internal/grpc/services/gateway/ocmshareprovider.go index bd7ebb76bb..ffad880f51 100644 --- a/internal/grpc/services/gateway/ocmshareprovider.go +++ b/internal/grpc/services/gateway/ocmshareprovider.go @@ -352,19 +352,25 @@ func (s *svc) createOCMReference(ctx context.Context, share *ocm.Share) (*rpc.St } log.Info().Msg("mount path will be:" + refPath) - createRefReq := &provider.CreateReferenceRequest{ - Ref: &provider.Reference{Path: refPath}, - TargetUri: targetURI, - } - c, err := s.findByPath(ctx, refPath) + c, p, err := s.findByPath(ctx, refPath) if err != nil { if _, ok := err.(errtypes.IsNotFound); ok { return status.NewNotFound(ctx, "storage provider not found"), nil } return status.NewInternal(ctx, err, "error finding storage provider"), nil } - + pRef, err := unwrap(&provider.Reference{Path: refPath}, p.ProviderPath) + if err != nil { + log.Err(err).Msg("gateway: error unwrapping") + return &rpc.Status{ + Code: rpc.Code_CODE_INTERNAL, + }, nil + } + createRefReq := &provider.CreateReferenceRequest{ + Ref: pRef, + TargetUri: targetURI, + } createRefRes, err := c.CreateReference(ctx, createRefReq) if err != nil { log.Err(err).Msg("gateway: error calling GetHome") diff --git a/internal/grpc/services/gateway/storageprovider.go b/internal/grpc/services/gateway/storageprovider.go index dcfd7be3e7..6a73ef3a2b 100644 --- a/internal/grpc/services/gateway/storageprovider.go +++ b/internal/grpc/services/gateway/storageprovider.go @@ -23,6 +23,7 @@ import ( "fmt" "net/url" "path" + "path/filepath" "strings" "sync" "time" @@ -81,7 +82,7 @@ func (s *svc) CreateHome(ctx context.Context, req *provider.CreateHomeRequest) ( log := appctx.GetLogger(ctx) home := s.getHome(ctx) - c, err := s.findByPath(ctx, home) + c, _, err := s.findByPath(ctx, home) if err != nil { return &provider.CreateHomeResponse{ Status: status.NewStatusFromErrType(ctx, "error finding home", err), @@ -101,7 +102,7 @@ func (s *svc) CreateHome(ctx context.Context, req *provider.CreateHomeRequest) ( func (s *svc) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) { log := appctx.GetLogger(ctx) // TODO: needs to be fixed - c, err := s.findByPath(ctx, "/users") + c, _, err := s.findByPath(ctx, "/users") if err != nil { return &provider.CreateStorageSpaceResponse{ Status: status.NewStatusFromErrType(ctx, "error finding path", err), @@ -249,7 +250,7 @@ func (s *svc) listStorageSpacesOnProvider(ctx context.Context, req *provider.Lis func (s *svc) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) { log := appctx.GetLogger(ctx) // TODO: needs to be fixed - c, err := s.find(ctx, &provider.Reference{ResourceId: req.StorageSpace.Root}) + c, _, err := s.find(ctx, &provider.Reference{ResourceId: req.StorageSpace.Root}) if err != nil { return &provider.UpdateStorageSpaceResponse{ Status: status.NewStatusFromErrType(ctx, "error finding ID", err), @@ -275,7 +276,7 @@ func (s *svc) DeleteStorageSpace(ctx context.Context, req *provider.DeleteStorag Status: status.NewInvalidArg(ctx, "space id must be separated by !"), }, nil } - c, err := s.find(ctx, &provider.Reference{ResourceId: &provider.ResourceId{ + c, _, err := s.find(ctx, &provider.Reference{ResourceId: &provider.ResourceId{ StorageId: storageid, OpaqueId: opaqeid, }}) @@ -472,13 +473,15 @@ func (s *svc) InitiateFileDownload(ctx context.Context, req *provider.InitiateFi func (s *svc) initiateFileDownload(ctx context.Context, req *provider.InitiateFileDownloadRequest) (*gateway.InitiateFileDownloadResponse, error) { // TODO(ishank011): enable downloading references spread across storage providers, eg. /eos - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &gateway.InitiateFileDownloadResponse{ Status: status.NewStatusFromErrType(ctx, "error initiating download ref="+req.Ref.String(), err), }, nil } - + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } storageRes, err := c.InitiateFileDownload(ctx, req) if err != nil { return nil, errors.Wrap(err, "gateway: error calling InitiateFileDownload") @@ -670,13 +673,15 @@ func (s *svc) InitiateFileUpload(ctx context.Context, req *provider.InitiateFile } func (s *svc) initiateFileUpload(ctx context.Context, req *provider.InitiateFileUploadRequest) (*gateway.InitiateFileUploadResponse, error) { - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &gateway.InitiateFileUploadResponse{ Status: status.NewStatusFromErrType(ctx, "initiateFileUpload ref="+req.Ref.String(), err), }, nil } - + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } storageRes, err := c.InitiateFileUpload(ctx, req) if err != nil { return nil, errors.Wrap(err, "gateway: error calling InitiateFileUpload") @@ -821,13 +826,15 @@ func (s *svc) CreateContainer(ctx context.Context, req *provider.CreateContainer } func (s *svc) createContainer(ctx context.Context, req *provider.CreateContainerRequest) (*provider.CreateContainerResponse, error) { - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &provider.CreateContainerResponse{ Status: status.NewStatusFromErrType(ctx, "createContainer ref="+req.Ref.String(), err), }, nil } - + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } res, err := c.CreateContainer(ctx, req) if err != nil { return nil, errors.Wrap(err, "gateway: error calling CreateContainer") @@ -977,13 +984,15 @@ func (s *svc) Delete(ctx context.Context, req *provider.DeleteRequest) (*provide func (s *svc) delete(ctx context.Context, req *provider.DeleteRequest) (*provider.DeleteResponse, error) { // TODO(ishank011): enable deleting references spread across storage providers, eg. /eos - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &provider.DeleteResponse{ Status: status.NewStatusFromErrType(ctx, "delete ref="+req.Ref.String(), err), }, nil } - + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } res, err := c.Delete(ctx, req) if err != nil { return nil, errors.Wrap(err, "gateway: error calling Delete") @@ -1146,19 +1155,27 @@ func (s *svc) move(ctx context.Context, req *provider.MoveRequest) (*provider.Mo Status: status.NewInternal(ctx, err, "error connecting to storage provider="+srcProvider.Address), }, nil } + if req.Source, err = unwrap(req.Source, srcProvider.ProviderPath); err != nil { + return nil, err + } + if req.Destination, err = unwrap(req.Destination, dstProvider.ProviderPath); err != nil { + return nil, err + } return c.Move(ctx, req) } func (s *svc) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitraryMetadataRequest) (*provider.SetArbitraryMetadataResponse, error) { // TODO(ishank011): enable for references spread across storage providers, eg. /eos - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &provider.SetArbitraryMetadataResponse{ Status: status.NewStatusFromErrType(ctx, "SetArbitraryMetadata ref="+req.Ref.String(), err), }, nil } - + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } res, err := c.SetArbitraryMetadata(ctx, req) if err != nil { return nil, errors.Wrap(err, "gateway: error calling Stat") @@ -1169,13 +1186,15 @@ func (s *svc) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitra func (s *svc) UnsetArbitraryMetadata(ctx context.Context, req *provider.UnsetArbitraryMetadataRequest) (*provider.UnsetArbitraryMetadataResponse, error) { // TODO(ishank011): enable for references spread across storage providers, eg. /eos - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &provider.UnsetArbitraryMetadataResponse{ Status: status.NewStatusFromErrType(ctx, "UnsetArbitraryMetadata ref="+req.Ref.String(), err), }, nil } - + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } res, err := c.UnsetArbitraryMetadata(ctx, req) if err != nil { return nil, errors.Wrap(err, "gateway: error calling Stat") @@ -1292,10 +1311,23 @@ func (s *svc) stat(ctx context.Context, req *provider.StatRequest) (*provider.St Status: status.NewInternal(ctx, err, "error connecting to storage provider="+providers[0].Address), }, nil } - rsp, err := c.Stat(ctx, req) + ref, err := unwrap(req.Ref, providers[0].ProviderPath) + if err != nil { + return nil, err + } + // We need to copy the request because we don't want to overwrite the original request + sReq := &provider.StatRequest{ + Opaque: req.Opaque, + Ref: ref, + ArbitraryMetadataKeys: req.ArbitraryMetadataKeys, + } + rsp, err := c.Stat(ctx, sReq) if err != nil || rsp.Status.Code != rpc.Code_CODE_OK { return rsp, err } + if rsp.Info != nil && utils.IsAbsoluteReference(req.Ref) { + wrap(rsp.Info, providers[0]) + } return rsp, nil } @@ -1323,7 +1355,8 @@ func (s *svc) statAcrossProviders(ctx context.Context, req *provider.StatRequest log.Err(err).Msg("error connecting to storage provider=" + p.Address) continue } - resp, err := c.Stat(ctx, req) + + resp, err := c.Stat(ctx, &provider.StatRequest{Opaque: req.Opaque, Ref: &provider.Reference{Path: "/"}, ArbitraryMetadataKeys: req.ArbitraryMetadataKeys}) if err != nil { log.Err(err).Msgf("gateway: error calling Stat %s: %+v", req.Ref.String(), p) continue @@ -1333,6 +1366,7 @@ func (s *svc) statAcrossProviders(ctx context.Context, req *provider.StatRequest continue } if resp.Info != nil { + wrap(resp.Info, p) info.Size += resp.Info.Size info.Mtime = utils.LaterTS(info.Mtime, resp.Info.Mtime) } @@ -1573,7 +1607,7 @@ func (s *svc) ListContainerStream(_ *provider.ListContainerStreamRequest, _ gate func (s *svc) listHome(ctx context.Context, req *provider.ListContainerRequest) (*provider.ListContainerResponse, error) { lcr, err := s.listContainer(ctx, &provider.ListContainerRequest{ - Ref: &provider.Reference{Path: s.getHome(ctx)}, + Ref: &provider.Reference{Path: "/"}, ArbitraryMetadataKeys: req.ArbitraryMetadataKeys, }) if err != nil { @@ -1685,7 +1719,8 @@ func (s *svc) listContainerAcrossProviders(ctx context.Context, req *provider.Li log.Err(err).Msg("error connecting to storage provider=" + p.Address) continue } - resp, err := c.ListContainer(ctx, req) + + resp, err := c.ListContainer(ctx, &provider.ListContainerRequest{Opaque: req.Opaque, Ref: &provider.Reference{Path: "/"}, ArbitraryMetadataKeys: req.ArbitraryMetadataKeys}) if err != nil { log.Err(err).Msgf("gateway: error calling Stat %s: %+v", req.Ref.String(), p) continue @@ -1695,8 +1730,9 @@ func (s *svc) listContainerAcrossProviders(ctx context.Context, req *provider.Li continue } - for _, info := range resp.Infos { - if p, ok := nestedInfos[info.Path]; ok { + infos := s.listVirtualView(ctx, req.Ref, resp.Infos, p) + for i := range infos { + if p, ok := nestedInfos[infos[i].Path]; ok { // Since more than one providers contribute to this path, // use a generic ID p.Id = &provider.ResourceId{ @@ -1704,12 +1740,12 @@ func (s *svc) listContainerAcrossProviders(ctx context.Context, req *provider.Li OpaqueId: uuid.New().String(), } // TODO(ishank011): aggregrate properties such as etag, checksum, etc. - p.Size += info.Size - p.Mtime = utils.LaterTS(p.Mtime, info.Mtime) + p.Size += infos[i].Size + p.Mtime = utils.LaterTS(p.Mtime, infos[i].Mtime) p.Type = provider.ResourceType_RESOURCE_TYPE_CONTAINER p.MimeType = "httpd/unix-directory" } else { - nestedInfos[info.Path] = info + nestedInfos[infos[i].Path] = infos[i] } } } @@ -1725,6 +1761,52 @@ func (s *svc) listContainerAcrossProviders(ctx context.Context, req *provider.Li }, nil } +func (s *svc) listVirtualView(ctx context.Context, ref *provider.Reference, mds []*provider.ResourceInfo, p *registry.ProviderInfo) []*provider.ResourceInfo { + nestedInfos := make(map[string]*provider.ResourceInfo) + infos := make([]*provider.ResourceInfo, 0, len(mds)) + + for _, info := range mds { + // Get the path prefixed with the mount point + wrap(info, p) + + // If info is an immediate child of the path in request, just use that + if path.Dir(info.Path) == path.Clean(ref.Path) { + infos = append(infos, info) + continue + } + + // info is a nested resource, so link it to its parent closest to the path in request + rel, err := filepath.Rel(ref.Path, info.Path) + if err != nil { + continue + } + + parent := path.Join(ref.Path, strings.Split(rel, "/")[0]) + + if p, ok := nestedInfos[parent]; ok { + p.Size += info.Size + p.Mtime = utils.LaterTS(p.Mtime, info.Mtime) + } else { + nestedInfos[parent] = &provider.ResourceInfo{ + Path: parent, + Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER, + Id: &provider.ResourceId{ + OpaqueId: uuid.New().String(), + }, + Size: info.Size, + Mtime: info.Mtime, + MimeType: "httpd/unix-directory", + } + } + } + + for _, info := range nestedInfos { + infos = append(infos, info) + } + + return infos +} + func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequest) (*provider.ListContainerResponse, error) { log := appctx.GetLogger(ctx) @@ -2005,13 +2087,16 @@ func (s *svc) CreateSymlink(ctx context.Context, req *provider.CreateSymlinkRequ } func (s *svc) ListFileVersions(ctx context.Context, req *provider.ListFileVersionsRequest) (*provider.ListFileVersionsResponse, error) { - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &provider.ListFileVersionsResponse{ Status: status.NewStatusFromErrType(ctx, "ListFileVersions ref="+req.Ref.String(), err), }, nil } - + req.Ref, err = unwrap(req.Ref, p.ProviderPath) + if err != nil { + return nil, err + } res, err := c.ListFileVersions(ctx, req) if err != nil { return nil, errors.Wrap(err, "gateway: error calling ListFileVersions") @@ -2021,13 +2106,15 @@ func (s *svc) ListFileVersions(ctx context.Context, req *provider.ListFileVersio } func (s *svc) RestoreFileVersion(ctx context.Context, req *provider.RestoreFileVersionRequest) (*provider.RestoreFileVersionResponse, error) { - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &provider.RestoreFileVersionResponse{ Status: status.NewStatusFromErrType(ctx, "RestoreFileVersion ref="+req.Ref.String(), err), }, nil } - + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } res, err := c.RestoreFileVersion(ctx, req) if err != nil { return nil, errors.Wrap(err, "gateway: error calling RestoreFileVersion") @@ -2042,14 +2129,21 @@ func (s *svc) ListRecycleStream(_ *provider.ListRecycleStreamRequest, _ gateway. // TODO use the ListRecycleRequest.Ref to only list the trash of a specific storage func (s *svc) ListRecycle(ctx context.Context, req *provider.ListRecycleRequest) (*provider.ListRecycleResponse, error) { - c, err := s.find(ctx, req.GetRef()) + c, p, err := s.find(ctx, req.GetRef()) if err != nil { return &provider.ListRecycleResponse{ Status: status.NewStatusFromErrType(ctx, "ListFileVersions ref="+req.Ref.String(), err), }, nil } - - res, err := c.ListRecycle(ctx, req) + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } + res, err := c.ListRecycle(ctx, &provider.ListRecycleRequest{ + Opaque: req.Opaque, + FromTs: req.FromTs, + ToTs: req.ToTs, + Ref: req.Ref, + }) if err != nil { return nil, errors.Wrap(err, "gateway: error calling ListRecycleRequest") } @@ -2058,13 +2152,15 @@ func (s *svc) ListRecycle(ctx context.Context, req *provider.ListRecycleRequest) } func (s *svc) RestoreRecycleItem(ctx context.Context, req *provider.RestoreRecycleItemRequest) (*provider.RestoreRecycleItemResponse, error) { - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &provider.RestoreRecycleItemResponse{ Status: status.NewStatusFromErrType(ctx, "RestoreRecycleItem ref="+req.Ref.String(), err), }, nil } - + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } res, err := c.RestoreRecycleItem(ctx, req) if err != nil { return nil, errors.Wrap(err, "gateway: error calling RestoreRecycleItem") @@ -2074,14 +2170,19 @@ func (s *svc) RestoreRecycleItem(ctx context.Context, req *provider.RestoreRecyc } func (s *svc) PurgeRecycle(ctx context.Context, req *provider.PurgeRecycleRequest) (*provider.PurgeRecycleResponse, error) { - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &provider.PurgeRecycleResponse{ Status: status.NewStatusFromErrType(ctx, "PurgeRecycle ref="+req.Ref.String(), err), }, nil } - - res, err := c.PurgeRecycle(ctx, req) + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } + res, err := c.PurgeRecycle(ctx, &provider.PurgeRecycleRequest{ + Opaque: req.GetOpaque(), + Ref: req.GetRef(), + }) if err != nil { return nil, errors.Wrap(err, "gateway: error calling PurgeRecycle") } @@ -2089,13 +2190,15 @@ func (s *svc) PurgeRecycle(ctx context.Context, req *provider.PurgeRecycleReques } func (s *svc) GetQuota(ctx context.Context, req *gateway.GetQuotaRequest) (*provider.GetQuotaResponse, error) { - c, err := s.find(ctx, req.Ref) + c, p, err := s.find(ctx, req.Ref) if err != nil { return &provider.GetQuotaResponse{ Status: status.NewStatusFromErrType(ctx, "GetQuota ref="+req.Ref.String(), err), }, nil } - + if req.Ref, err = unwrap(req.Ref, p.ProviderPath); err != nil { + return nil, err + } res, err := c.GetQuota(ctx, &provider.GetQuotaRequest{ Opaque: req.GetOpaque(), Ref: req.GetRef(), @@ -2106,17 +2209,19 @@ func (s *svc) GetQuota(ctx context.Context, req *gateway.GetQuotaRequest) (*prov return res, nil } -func (s *svc) findByPath(ctx context.Context, path string) (provider.ProviderAPIClient, error) { +func (s *svc) findByPath(ctx context.Context, path string) (provider.ProviderAPIClient, *registry.ProviderInfo, error) { ref := &provider.Reference{Path: path} return s.find(ctx, ref) } -func (s *svc) find(ctx context.Context, ref *provider.Reference) (provider.ProviderAPIClient, error) { +func (s *svc) find(ctx context.Context, ref *provider.Reference) (provider.ProviderAPIClient, *registry.ProviderInfo, error) { p, err := s.findProviders(ctx, ref) if err != nil { - return nil, err + return nil, nil, err } - return s.getStorageProviderClient(ctx, p[0]) + + client, err := s.getStorageProviderClient(ctx, p[0]) + return client, p[0], err } func (s *svc) getStorageProviderClient(_ context.Context, p *registry.ProviderInfo) (provider.ProviderAPIClient, error) { @@ -2166,13 +2271,13 @@ func (s *svc) findProviders(ctx context.Context, ref *provider.Reference) ([]*re } func getUniqueProviders(providers []*registry.ProviderInfo) []*registry.ProviderInfo { - unique := make(map[string]bool) + unique := make(map[string]*registry.ProviderInfo) for _, p := range providers { - unique[p.Address] = true + unique[p.Address] = p } p := make([]*registry.ProviderInfo, 0, len(unique)) - for addr := range unique { - p = append(p, ®istry.ProviderInfo{Address: addr}) + for _, providerInfo := range unique { + p = append(p, providerInfo) } return p } @@ -2181,3 +2286,28 @@ type etagWithTS struct { Etag string Timestamp time.Time } + +func unwrap(ref *provider.Reference, providerPath string) (*provider.Reference, error) { + // all references with an id can be passed on to the driver + // there are two cases: + // 1. absolute id references (resource_id is set, path is empty) + // 2. relative references (resource_id is set, path starts with a `.`) + if ref.GetResourceId() != nil { + return ref, nil + } + + if !strings.HasPrefix(ref.GetPath(), "/") { + // abort, absolute path references must start with a `/` + return nil, errtypes.BadRequest("ref is invalid: " + ref.String()) + } + + p := strings.TrimPrefix(ref.Path, providerPath) + if p == "" { + p = "/" + } + return &provider.Reference{Path: p}, nil +} + +func wrap(ri *provider.ResourceInfo, providerInfo *registry.ProviderInfo) { + ri.Path = path.Join(providerInfo.ProviderPath, ri.Path) +} diff --git a/internal/grpc/services/gateway/usershareprovider.go b/internal/grpc/services/gateway/usershareprovider.go index 7511e0a581..de268a9e0a 100644 --- a/internal/grpc/services/gateway/usershareprovider.go +++ b/internal/grpc/services/gateway/usershareprovider.go @@ -369,7 +369,7 @@ func (s *svc) removeReference(ctx context.Context, resourceID *provider.Resource log := appctx.GetLogger(ctx) idReference := &provider.Reference{ResourceId: resourceID} - storageProvider, err := s.find(ctx, idReference) + storageProvider, _, err := s.find(ctx, idReference) if err != nil { if _, ok := err.(errtypes.IsNotFound); ok { return status.NewNotFound(ctx, "storage provider not found") @@ -397,13 +397,18 @@ func (s *svc) removeReference(ctx context.Context, resourceID *provider.Resource sharePath := path.Join(homeRes.Path, s.c.ShareFolder, path.Base(statRes.Info.Path)) log.Debug().Str("share_path", sharePath).Msg("remove reference of share") - homeProvider, err := s.find(ctx, &provider.Reference{Path: sharePath}) + sharePathRef := &provider.Reference{Path: sharePath} + homeProvider, providerInfo, err := s.find(ctx, sharePathRef) if err != nil { if _, ok := err.(errtypes.IsNotFound); ok { return status.NewNotFound(ctx, "storage provider not found") } return status.NewInternal(ctx, err, "error finding storage provider") } + ref, err := unwrap(sharePathRef, providerInfo.ProviderPath) + if err != nil { + return status.NewInternal(ctx, err, "could not unwrap share path reference") + } deleteReq := &provider.DeleteRequest{ Opaque: &typesv1beta1.Opaque{ @@ -412,7 +417,7 @@ func (s *svc) removeReference(ctx context.Context, resourceID *provider.Resource "deleting_shared_resource": {}, }, }, - Ref: &provider.Reference{Path: sharePath}, + Ref: ref, } deleteResp, err := homeProvider.Delete(ctx, deleteReq) @@ -443,7 +448,7 @@ func (s *svc) createReference(ctx context.Context, resourceID *provider.Resource log := appctx.GetLogger(ctx) // get the metadata about the share - c, err := s.find(ctx, ref) + c, _, err := s.find(ctx, ref) if err != nil { if _, ok := err.(errtypes.IsNotFound); ok { return status.NewNotFound(ctx, "storage provider not found") @@ -485,20 +490,25 @@ func (s *svc) createReference(ctx context.Context, resourceID *provider.Resource refPath := path.Join(homeRes.Path, s.c.ShareFolder, path.Base(statRes.Info.Path)) log.Info().Msg("mount path will be:" + refPath) - createRefReq := &provider.CreateReferenceRequest{ - Ref: &provider.Reference{Path: refPath}, - // cs3 is the Scheme and %s/%s is the Opaque parts of a net.URL. - TargetUri: fmt.Sprintf("cs3:%s/%s", resourceID.GetStorageId(), resourceID.GetOpaqueId()), - } - - c, err = s.findByPath(ctx, refPath) + c, p, err := s.findByPath(ctx, refPath) if err != nil { if _, ok := err.(errtypes.IsNotFound); ok { return status.NewNotFound(ctx, "storage provider not found") } return status.NewInternal(ctx, err, "error finding storage provider") } - + pRef, err := unwrap(&provider.Reference{Path: refPath}, p.ProviderPath) + if err != nil { + log.Err(err).Msg("gateway: error unwrapping reference") + return &rpc.Status{ + Code: rpc.Code_CODE_INTERNAL, + } + } + createRefReq := &provider.CreateReferenceRequest{ + Ref: pRef, + // cs3 is the Scheme and %s/%s is the Opaque parts of a net.URL. + TargetUri: fmt.Sprintf("cs3:%s/%s", resourceID.GetStorageId(), resourceID.GetOpaqueId()), + } createRefRes, err := c.CreateReference(ctx, createRefReq) if err != nil { log.Err(err).Msg("gateway: error calling GetHome") @@ -525,7 +535,7 @@ func (s *svc) denyGrant(ctx context.Context, id *provider.ResourceId, g *provide Grantee: g, } - c, err := s.find(ctx, ref) + c, _, err := s.find(ctx, ref) if err != nil { if _, ok := err.(errtypes.IsNotFound); ok { return status.NewNotFound(ctx, "storage provider not found"), nil @@ -558,7 +568,7 @@ func (s *svc) addGrant(ctx context.Context, id *provider.ResourceId, g *provider }, } - c, err := s.find(ctx, ref) + c, _, err := s.find(ctx, ref) if err != nil { if _, ok := err.(errtypes.IsNotFound); ok { return status.NewNotFound(ctx, "storage provider not found"), nil @@ -590,7 +600,7 @@ func (s *svc) updateGrant(ctx context.Context, id *provider.ResourceId, g *provi }, } - c, err := s.find(ctx, ref) + c, _, err := s.find(ctx, ref) if err != nil { if _, ok := err.(errtypes.IsNotFound); ok { return status.NewNotFound(ctx, "storage provider not found"), nil @@ -623,7 +633,7 @@ func (s *svc) removeGrant(ctx context.Context, id *provider.ResourceId, g *provi }, } - c, err := s.find(ctx, ref) + c, _, err := s.find(ctx, ref) if err != nil { if _, ok := err.(errtypes.IsNotFound); ok { return status.NewNotFound(ctx, "storage provider not found"), nil diff --git a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go index 9132811bb1..c3552933e9 100644 --- a/internal/grpc/services/publicstorageprovider/publicstorageprovider.go +++ b/internal/grpc/services/publicstorageprovider/publicstorageprovider.go @@ -49,14 +49,12 @@ func init() { } type config struct { - MountPath string `mapstructure:"mount_path"` GatewayAddr string `mapstructure:"gateway_addr"` } type service struct { - conf *config - mountPath string - gateway gateway.GatewayAPIClient + conf *config + gateway gateway.GatewayAPIClient } func (s *service) Close() error { @@ -87,17 +85,14 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return nil, err } - mountPath := c.MountPath - gateway, err := pool.GetGatewayServiceClient(c.GatewayAddr) if err != nil { return nil, err } service := &service{ - conf: c, - mountPath: mountPath, - gateway: gateway, + conf: c, + gateway: gateway, } return service, nil @@ -495,7 +490,7 @@ func (s *service) Stat(ctx context.Context, req *provider.StatRequest) (*provide if err := addShare(statResponse.Info, ls); err != nil { appctx.GetLogger(ctx).Error().Err(err).Interface("share", ls).Interface("info", statResponse.Info).Msg("error when adding share") } - statResponse.Info.Path = path.Join(s.mountPath, "/", tkn, relativePath) + statResponse.Info.Path = path.Join("/", tkn, relativePath) filterPermissions(statResponse.Info.PermissionSet, ls.GetPermissions().Permissions) } @@ -554,7 +549,7 @@ func (s *service) ListContainer(ctx context.Context, req *provider.ListContainer for i := range listContainerR.Infos { filterPermissions(listContainerR.Infos[i].PermissionSet, ls.GetPermissions().Permissions) - listContainerR.Infos[i].Path = path.Join(s.mountPath, "/", tkn, relativePath, path.Base(listContainerR.Infos[i].Path)) + listContainerR.Infos[i].Path = path.Join("/", tkn, relativePath, path.Base(listContainerR.Infos[i].Path)) if err := addShare(listContainerR.Infos[i], ls); err != nil { appctx.GetLogger(ctx).Error().Err(err).Interface("share", ls).Interface("info", listContainerR.Infos[i]).Msg("error when adding share") } @@ -594,15 +589,7 @@ func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (token st return "", "", errtypes.BadRequest("invalid ref: " + ref.String()) } - // i.e path: /public/{token}/path/to/subfolders - fn := ref.GetPath() - // fsfn: /{token}/path/to/subfolders - fsfn, err := s.trimMountPrefix(fn) - if err != nil { - return "", "", err - } - - parts := strings.SplitN(fsfn, "/", 3) + parts := strings.SplitN(ref.Path, "/", 3) token = parts[1] if len(parts) > 2 { relativePath = parts[2] @@ -667,13 +654,6 @@ func (s *service) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) ( return nil, gstatus.Errorf(codes.Unimplemented, "method not implemented") } -func (s *service) trimMountPrefix(fn string) (string, error) { - if strings.HasPrefix(fn, s.mountPath) { - return path.Join("/", strings.TrimPrefix(fn, s.mountPath)), nil - } - return "", errors.Errorf("path=%q does not belong to this storage provider mount path=%q"+fn, s.mountPath) -} - // resolveToken returns the path and share for the publicly shared resource. func (s *service) resolveToken(ctx context.Context, token string) (string, *link.PublicShare, *provider.ResourceInfo, *rpc.Status, error) { driver, err := pool.GetGatewayServiceClient(s.conf.GatewayAddr) diff --git a/internal/grpc/services/storageprovider/storageprovider.go b/internal/grpc/services/storageprovider/storageprovider.go index a7e067147b..46090ab508 100644 --- a/internal/grpc/services/storageprovider/storageprovider.go +++ b/internal/grpc/services/storageprovider/storageprovider.go @@ -25,10 +25,8 @@ import ( "net/url" "os" "path" - "path/filepath" "sort" "strconv" - "strings" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -42,7 +40,6 @@ import ( "github.com/cs3org/reva/pkg/storage/fs/registry" rtrace "github.com/cs3org/reva/pkg/trace" "github.com/cs3org/reva/pkg/utils" - "github.com/google/uuid" "github.com/mitchellh/mapstructure" "github.com/pkg/errors" "go.opentelemetry.io/otel/attribute" @@ -54,7 +51,6 @@ func init() { } type config struct { - MountPath string `mapstructure:"mount_path" docs:"/;The path where the file system would be mounted."` MountID string `mapstructure:"mount_id" docs:"-;The ID of the mounted file system."` Driver string `mapstructure:"driver" docs:"localhome;The storage driver to be used."` Drivers map[string]map[string]interface{} `mapstructure:"drivers" docs:"url:pkg/storage/fs/localhome/localhome.go"` @@ -70,10 +66,6 @@ func (c *config) init() { c.Driver = "localhome" } - if c.MountPath == "" { - c.MountPath = "/" - } - if c.MountID == "" { c.MountID = "00000000-0000-0000-0000-000000000000" } @@ -102,12 +94,12 @@ func (c *config) init() { } type service struct { - conf *config - storage storage.FS - mountPath, mountID string - tmpFolder string - dataServerURL *url.URL - availableXS []*provider.ResourceChecksumPriority + conf *config + storage storage.FS + mountID string + tmpFolder string + dataServerURL *url.URL + availableXS []*provider.ResourceChecksumPriority } func (s *service) Close() error { @@ -159,7 +151,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { return nil, err } - mountPath := c.MountPath mountID := c.MountID fs, err := getFS(c) @@ -189,7 +180,6 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { conf: c, storage: fs, tmpFolder: c.TmpFolder, - mountPath: mountPath, mountID: mountID, dataServerURL: u, availableXS: xsTypes, @@ -205,15 +195,7 @@ func registerMimeTypes(mimes map[string]string) { } func (s *service) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitraryMetadataRequest) (*provider.SetArbitraryMetadataResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - err := errors.Wrap(err, "storageprovidersvc: error unwrapping path") - return &provider.SetArbitraryMetadataResponse{ - Status: status.NewInternal(ctx, err, "error setting arbitrary metadata"), - }, nil - } - - if err := s.storage.SetArbitraryMetadata(ctx, newRef, req.ArbitraryMetadata); err != nil { + if err := s.storage.SetArbitraryMetadata(ctx, req.Ref, req.ArbitraryMetadata); err != nil { var st *rpc.Status switch err.(type) { case errtypes.IsNotFound: @@ -235,15 +217,7 @@ func (s *service) SetArbitraryMetadata(ctx context.Context, req *provider.SetArb } func (s *service) UnsetArbitraryMetadata(ctx context.Context, req *provider.UnsetArbitraryMetadataRequest) (*provider.UnsetArbitraryMetadataResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - err := errors.Wrap(err, "storageprovidersvc: error unwrapping path") - return &provider.UnsetArbitraryMetadataResponse{ - Status: status.NewInternal(ctx, err, "error unsetting arbitrary metadata"), - }, nil - } - - if err := s.storage.UnsetArbitraryMetadata(ctx, newRef, req.ArbitraryMetadataKeys); err != nil { + if err := s.storage.UnsetArbitraryMetadata(ctx, req.Ref, req.ArbitraryMetadataKeys); err != nil { var st *rpc.Status switch err.(type) { case errtypes.IsNotFound: @@ -280,16 +254,10 @@ func (s *service) InitiateFileDownload(ctx context.Context, req *provider.Initia protocol.Protocol = "spaces" u.Path = path.Join(u.Path, "spaces", req.Ref.ResourceId.StorageId+"!"+req.Ref.ResourceId.OpaqueId, req.Ref.Path) } else { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.InitiateFileDownloadResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } // Currently, we only support the simple protocol for GET requests // Once we have multiple protocols, this would be moved to the fs layer protocol.Protocol = "simple" - u.Path = path.Join(u.Path, "simple", newRef.GetPath()) + u.Path = path.Join(u.Path, "simple", req.Ref.GetPath()) } protocol.DownloadEndpoint = u.String() @@ -303,13 +271,7 @@ func (s *service) InitiateFileDownload(ctx context.Context, req *provider.Initia func (s *service) InitiateFileUpload(ctx context.Context, req *provider.InitiateFileUploadRequest) (*provider.InitiateFileUploadResponse, error) { // TODO(labkode): same considerations as download log := appctx.GetLogger(ctx) - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.InitiateFileUploadResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - if newRef.GetPath() == "/" { + if req.Ref.GetPath() == "/" { return &provider.InitiateFileUploadResponse{ Status: status.NewInternal(ctx, errtypes.BadRequest("can't upload to mount path"), "can't upload to mount path"), }, nil @@ -336,7 +298,7 @@ func (s *service) InitiateFileUpload(ctx context.Context, req *provider.Initiate metadata["mtime"] = string(req.Opaque.Map["X-OC-Mtime"].Value) } } - uploadIDs, err := s.storage.InitiateUpload(ctx, newRef, uploadLength, metadata) + uploadIDs, err := s.storage.InitiateUpload(ctx, req.Ref, uploadLength, metadata) if err != nil { var st *rpc.Status switch err.(type) { @@ -398,8 +360,6 @@ func (s *service) GetPath(ctx context.Context, req *provider.GetPathRequest) (*p Status: status.NewInternal(ctx, err, "error getting path by id"), }, nil } - - fn = path.Join(s.mountPath, path.Clean(fn)) res := &provider.GetPathResponse{ Path: fn, Status: status.NewOK(ctx), @@ -408,11 +368,9 @@ func (s *service) GetPath(ctx context.Context, req *provider.GetPathRequest) (*p } func (s *service) GetHome(ctx context.Context, req *provider.GetHomeRequest) (*provider.GetHomeResponse, error) { - home := path.Join(s.mountPath) - res := &provider.GetHomeResponse{ Status: status.NewOK(ctx), - Path: home, + Path: "/", } return res, nil @@ -515,13 +473,7 @@ func (s *service) DeleteStorageSpace(ctx context.Context, req *provider.DeleteSt } func (s *service) CreateContainer(ctx context.Context, req *provider.CreateContainerRequest) (*provider.CreateContainerResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.CreateContainerResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - if err := s.storage.CreateDir(ctx, newRef); err != nil { + if err := s.storage.CreateDir(ctx, req.Ref); err != nil { var st *rpc.Status switch err.(type) { case errtypes.IsNotFound: @@ -545,13 +497,7 @@ func (s *service) CreateContainer(ctx context.Context, req *provider.CreateConta } func (s *service) Delete(ctx context.Context, req *provider.DeleteRequest) (*provider.DeleteResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.DeleteResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - if newRef.GetPath() == "/" { + if req.Ref.GetPath() == "/" { return &provider.DeleteResponse{ Status: status.NewInternal(ctx, errtypes.BadRequest("can't delete mount path"), "can't delete mount path"), }, nil @@ -566,7 +512,7 @@ func (s *service) Delete(ctx context.Context, req *provider.DeleteRequest) (*pro } } - if err := s.storage.Delete(ctx, newRef); err != nil { + if err := s.storage.Delete(ctx, req.Ref); err != nil { var st *rpc.Status switch err.(type) { case errtypes.IsNotFound: @@ -588,20 +534,7 @@ func (s *service) Delete(ctx context.Context, req *provider.DeleteRequest) (*pro } func (s *service) Move(ctx context.Context, req *provider.MoveRequest) (*provider.MoveResponse, error) { - sourceRef, err := s.unwrap(ctx, req.Source) - if err != nil { - return &provider.MoveResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping source path"), - }, nil - } - targetRef, err := s.unwrap(ctx, req.Destination) - if err != nil { - return &provider.MoveResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping destination path"), - }, nil - } - - if err := s.storage.Move(ctx, sourceRef, targetRef); err != nil { + if err := s.storage.Move(ctx, req.Source, req.Destination); err != nil { var st *rpc.Status switch err.(type) { case errtypes.IsNotFound: @@ -609,7 +542,7 @@ func (s *service) Move(ctx context.Context, req *provider.MoveRequest) (*provide case errtypes.PermissionDenied: st = status.NewPermissionDenied(ctx, err, "permission denied") default: - st = status.NewInternal(ctx, err, "error moving: "+sourceRef.String()) + st = status.NewInternal(ctx, err, "error moving: "+req.Source.String()) } return &provider.MoveResponse{ Status: st, @@ -631,15 +564,7 @@ func (s *service) Stat(ctx context.Context, req *provider.StatRequest) (*provide Value: attribute.StringValue(req.Ref.String()), }) - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - // The path might be a virtual view; handle that case - if utils.IsAbsolutePathReference(req.Ref) && strings.HasPrefix(s.mountPath, req.Ref.Path) { - return s.statVirtualView(ctx, req.Ref) - } - } - - md, err := s.storage.GetMD(ctx, newRef, req.ArbitraryMetadataKeys) + md, err := s.storage.GetMD(ctx, req.Ref, req.ArbitraryMetadataKeys) if err != nil { var st *rpc.Status switch err.(type) { @@ -654,12 +579,8 @@ func (s *service) Stat(ctx context.Context, req *provider.StatRequest) (*provide Status: st, }, nil } + md.Id.StorageId = s.mountID - if err := s.wrap(ctx, md, utils.IsAbsoluteReference(req.Ref)); err != nil { - return &provider.StatResponse{ - Status: status.NewInternal(ctx, err, "error wrapping path"), - }, nil - } res := &provider.StatResponse{ Status: status.NewOK(ctx), Info: md, @@ -667,57 +588,11 @@ func (s *service) Stat(ctx context.Context, req *provider.StatRequest) (*provide return res, nil } -func (s *service) statVirtualView(ctx context.Context, ref *provider.Reference) (*provider.StatResponse, error) { - // The reference in the request encompasses this provider - // So we need to stat root, and update the required path - md, err := s.storage.GetMD(ctx, &provider.Reference{Path: "/"}, []string{}) - if err != nil { - var st *rpc.Status - switch err.(type) { - case errtypes.IsNotFound: - st = status.NewNotFound(ctx, "path not found when statting") - case errtypes.PermissionDenied: - st = status.NewPermissionDenied(ctx, err, "permission denied") - default: - st = status.NewInternal(ctx, err, "error statting root") - } - return &provider.StatResponse{ - Status: st, - }, nil - } - - if err := s.wrap(ctx, md, true); err != nil { - return &provider.StatResponse{ - Status: status.NewInternal(ctx, err, "error wrapping path"), - }, nil - } - - // Don't expose the underlying path - md.Path = ref.Path - - return &provider.StatResponse{ - Status: status.NewOK(ctx), - Info: md, - }, nil -} - func (s *service) ListContainerStream(req *provider.ListContainerStreamRequest, ss provider.ProviderAPI_ListContainerStreamServer) error { ctx := ss.Context() log := appctx.GetLogger(ctx) - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - res := &provider.ListContainerStreamResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - } - if err := ss.Send(res); err != nil { - log.Error().Err(err).Msg("ListContainerStream: error sending response") - return err - } - return nil - } - - mds, err := s.storage.ListFolder(ctx, newRef, req.ArbitraryMetadataKeys) + mds, err := s.storage.ListFolder(ctx, req.Ref, req.ArbitraryMetadataKeys) if err != nil { var st *rpc.Status switch err.(type) { @@ -738,18 +613,8 @@ func (s *service) ListContainerStream(req *provider.ListContainerStreamRequest, return nil } - prefixMountpoint := utils.IsAbsoluteReference(req.Ref) for _, md := range mds { - if err := s.wrap(ctx, md, prefixMountpoint); err != nil { - res := &provider.ListContainerStreamResponse{ - Status: status.NewInternal(ctx, err, "error wrapping path"), - } - if err := ss.Send(res); err != nil { - log.Error().Err(err).Msg("ListContainerStream: error sending response") - return err - } - return nil - } + md.Id.StorageId = s.mountID res := &provider.ListContainerStreamResponse{ Info: md, Status: status.NewOK(ctx), @@ -764,19 +629,7 @@ func (s *service) ListContainerStream(req *provider.ListContainerStreamRequest, } func (s *service) ListContainer(ctx context.Context, req *provider.ListContainerRequest) (*provider.ListContainerResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - // The path might be a virtual view; handle that case - if utils.IsAbsolutePathReference(req.Ref) && strings.HasPrefix(s.mountPath, req.Ref.Path) { - return s.listVirtualView(ctx, req.Ref) - } - - return &provider.ListContainerResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - - mds, err := s.storage.ListFolder(ctx, newRef, req.ArbitraryMetadataKeys) + mds, err := s.storage.ListFolder(ctx, req.Ref, req.ArbitraryMetadataKeys) if err != nil { var st *rpc.Status switch err.(type) { @@ -792,100 +645,19 @@ func (s *service) ListContainer(ctx context.Context, req *provider.ListContainer }, nil } - var infos = make([]*provider.ResourceInfo, 0, len(mds)) - prefixMountpoint := utils.IsAbsoluteReference(req.Ref) - for _, md := range mds { - if err := s.wrap(ctx, md, prefixMountpoint); err != nil { - return &provider.ListContainerResponse{ - Status: status.NewInternal(ctx, err, "error wrapping path"), - }, nil - } - infos = append(infos, md) + for i := range mds { + mds[i].Id.StorageId = s.mountID } + res := &provider.ListContainerResponse{ Status: status.NewOK(ctx), - Infos: infos, + Infos: mds, } return res, nil } -func (s *service) listVirtualView(ctx context.Context, ref *provider.Reference) (*provider.ListContainerResponse, error) { - // The reference in the request encompasses this provider - // So we need to list root, merge the responses and return only the immediate children - mds, err := s.storage.ListFolder(ctx, &provider.Reference{Path: "/"}, []string{}) - if err != nil { - var st *rpc.Status - switch err.(type) { - case errtypes.IsNotFound: - st = status.NewNotFound(ctx, "path not found when listing root") - case errtypes.PermissionDenied: - st = status.NewPermissionDenied(ctx, err, "permission denied") - default: - st = status.NewInternal(ctx, err, "error listing root") - } - return &provider.ListContainerResponse{ - Status: st, - }, nil - } - - nestedInfos := make(map[string]*provider.ResourceInfo) - infos := make([]*provider.ResourceInfo, 0, len(mds)) - - for _, info := range mds { - // Get the path prefixed with the mount point - if err := s.wrap(ctx, info, true); err != nil { - continue - } - - // If info is an immediate child of the path in request, just use that - if path.Dir(info.Path) == path.Clean(ref.Path) { - infos = append(infos, info) - continue - } - - // info is a nested resource, so link it to its parent closest to the path in request - rel, err := filepath.Rel(ref.Path, info.Path) - if err != nil { - continue - } - parent := path.Join(ref.Path, strings.Split(rel, "/")[0]) - - if p, ok := nestedInfos[parent]; ok { - p.Size += info.Size - p.Mtime = utils.LaterTS(p.Mtime, info.Mtime) - } else { - nestedInfos[parent] = &provider.ResourceInfo{ - Path: parent, - Type: provider.ResourceType_RESOURCE_TYPE_CONTAINER, - Id: &provider.ResourceId{ - OpaqueId: uuid.New().String(), - }, - Size: info.Size, - Mtime: info.Mtime, - MimeType: "httpd/unix-directory", - } - } - } - - for _, info := range nestedInfos { - infos = append(infos, info) - } - - return &provider.ListContainerResponse{ - Status: status.NewOK(ctx), - Infos: infos, - }, nil -} - func (s *service) ListFileVersions(ctx context.Context, req *provider.ListFileVersionsRequest) (*provider.ListFileVersionsResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.ListFileVersionsResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - - revs, err := s.storage.ListRevisions(ctx, newRef) + revs, err := s.storage.ListRevisions(ctx, req.Ref) if err != nil { var st *rpc.Status switch err.(type) { @@ -911,14 +683,7 @@ func (s *service) ListFileVersions(ctx context.Context, req *provider.ListFileVe } func (s *service) RestoreFileVersion(ctx context.Context, req *provider.RestoreFileVersionRequest) (*provider.RestoreFileVersionResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.RestoreFileVersionResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - - if err := s.storage.RestoreRevision(ctx, newRef, req.Key); err != nil { + if err := s.storage.RestoreRevision(ctx, req.Ref, req.Key); err != nil { var st *rpc.Status switch err.(type) { case errtypes.IsNotFound: @@ -943,13 +708,8 @@ func (s *service) ListRecycleStream(req *provider.ListRecycleStreamRequest, ss p ctx := ss.Context() log := appctx.GetLogger(ctx) - ref, err := s.unwrap(ctx, req.Ref) - if err != nil { - return err - } - key, itemPath := router.ShiftPath(req.Key) - items, err := s.storage.ListRecycle(ctx, ref.GetPath(), key, itemPath) + items, err := s.storage.ListRecycle(ctx, req.Ref.Path, key, itemPath) if err != nil { var st *rpc.Status switch err.(type) { @@ -985,12 +745,8 @@ func (s *service) ListRecycleStream(req *provider.ListRecycleStreamRequest, ss p } func (s *service) ListRecycle(ctx context.Context, req *provider.ListRecycleRequest) (*provider.ListRecycleResponse, error) { - ref, err := s.unwrap(ctx, req.Ref) - if err != nil { - return nil, err - } - key, itemPath := router.ShiftPath(req.Key) - items, err := s.storage.ListRecycle(ctx, ref.GetPath(), key, itemPath) + key, itemPath := router.ShiftPath(req.Ref.Path) + items, err := s.storage.ListRecycle(ctx, req.Ref.Path, key, itemPath) // TODO(labkode): CRITICAL: fill recycle info with storage provider. if err != nil { var st *rpc.Status @@ -1016,12 +772,8 @@ func (s *service) ListRecycle(ctx context.Context, req *provider.ListRecycleRequ func (s *service) RestoreRecycleItem(ctx context.Context, req *provider.RestoreRecycleItemRequest) (*provider.RestoreRecycleItemResponse, error) { // TODO(labkode): CRITICAL: fill recycle info with storage provider. - ref, err := s.unwrap(ctx, req.Ref) - if err != nil { - return nil, err - } key, itemPath := router.ShiftPath(req.Key) - if err := s.storage.RestoreRecycleItem(ctx, ref.GetPath(), key, itemPath, req.RestoreRef); err != nil { + if err := s.storage.RestoreRecycleItem(ctx, req.Ref.Path, key, itemPath, req.RestoreRef); err != nil { var st *rpc.Status switch err.(type) { case errtypes.IsNotFound: @@ -1043,14 +795,10 @@ func (s *service) RestoreRecycleItem(ctx context.Context, req *provider.RestoreR } func (s *service) PurgeRecycle(ctx context.Context, req *provider.PurgeRecycleRequest) (*provider.PurgeRecycleResponse, error) { - ref, err := s.unwrap(ctx, req.Ref) - if err != nil { - return nil, err - } // if a key was sent as opaque id purge only that item key, itemPath := router.ShiftPath(req.Key) if key != "" { - if err := s.storage.PurgeRecycleItem(ctx, ref.GetPath(), key, itemPath); err != nil { + if err := s.storage.PurgeRecycleItem(ctx, req.Ref.Path, key, itemPath); err != nil { var st *rpc.Status switch err.(type) { case errtypes.IsNotFound: @@ -1087,14 +835,7 @@ func (s *service) PurgeRecycle(ctx context.Context, req *provider.PurgeRecycleRe } func (s *service) ListGrants(ctx context.Context, req *provider.ListGrantsRequest) (*provider.ListGrantsResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.ListGrantsResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - - grants, err := s.storage.ListGrants(ctx, newRef) + grants, err := s.storage.ListGrants(ctx, req.Ref) if err != nil { var st *rpc.Status switch err.(type) { @@ -1118,13 +859,6 @@ func (s *service) ListGrants(ctx context.Context, req *provider.ListGrantsReques } func (s *service) DenyGrant(ctx context.Context, req *provider.DenyGrantRequest) (*provider.DenyGrantResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.DenyGrantResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - // check grantee type is valid if req.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_INVALID { return &provider.DenyGrantResponse{ @@ -1132,7 +866,7 @@ func (s *service) DenyGrant(ctx context.Context, req *provider.DenyGrantRequest) }, nil } - err = s.storage.DenyGrant(ctx, newRef, req.Grantee) + err := s.storage.DenyGrant(ctx, req.Ref, req.Grantee) if err != nil { var st *rpc.Status switch err.(type) { @@ -1155,13 +889,6 @@ func (s *service) DenyGrant(ctx context.Context, req *provider.DenyGrantRequest) } func (s *service) AddGrant(ctx context.Context, req *provider.AddGrantRequest) (*provider.AddGrantResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.AddGrantResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - // check grantee type is valid if req.Grant.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_INVALID { return &provider.AddGrantResponse{ @@ -1169,7 +896,7 @@ func (s *service) AddGrant(ctx context.Context, req *provider.AddGrantRequest) ( }, nil } - err = s.storage.AddGrant(ctx, newRef, req.Grant) + err := s.storage.AddGrant(ctx, req.Ref, req.Grant) if err != nil { var st *rpc.Status switch err.(type) { @@ -1199,14 +926,7 @@ func (s *service) UpdateGrant(ctx context.Context, req *provider.UpdateGrantRequ }, nil } - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.UpdateGrantResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - - if err := s.storage.UpdateGrant(ctx, newRef, req.Grant); err != nil { + if err := s.storage.UpdateGrant(ctx, req.Ref, req.Grant); err != nil { var st *rpc.Status switch err.(type) { case errtypes.IsNotFound: @@ -1235,14 +955,7 @@ func (s *service) RemoveGrant(ctx context.Context, req *provider.RemoveGrantRequ }, nil } - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.RemoveGrantResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - - if err := s.storage.RemoveGrant(ctx, newRef, req.Grant); err != nil { + if err := s.storage.RemoveGrant(ctx, req.Ref, req.Grant); err != nil { var st *rpc.Status switch err.(type) { case errtypes.IsNotFound: @@ -1275,14 +988,7 @@ func (s *service) CreateReference(ctx context.Context, req *provider.CreateRefer }, nil } - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.CreateReferenceResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - - if err := s.storage.CreateReference(ctx, newRef.GetPath(), u); err != nil { + if err := s.storage.CreateReference(ctx, req.Ref.GetPath(), u); err != nil { log.Err(err).Msg("error calling CreateReference") var st *rpc.Status switch err.(type) { @@ -1310,13 +1016,7 @@ func (s *service) CreateSymlink(ctx context.Context, req *provider.CreateSymlink } func (s *service) GetQuota(ctx context.Context, req *provider.GetQuotaRequest) (*provider.GetQuotaResponse, error) { - newRef, err := s.unwrap(ctx, req.Ref) - if err != nil { - return &provider.GetQuotaResponse{ - Status: status.NewInternal(ctx, err, "error unwrapping path"), - }, nil - } - total, used, err := s.storage.GetQuota(ctx, newRef) + total, used, err := s.storage.GetQuota(ctx, req.Ref) if err != nil { var st *rpc.Status switch err.(type) { @@ -1347,47 +1047,6 @@ func getFS(c *config) (storage.FS, error) { return nil, errtypes.NotFound("driver not found: " + c.Driver) } -func (s *service) unwrap(ctx context.Context, ref *provider.Reference) (*provider.Reference, error) { - // all references with an id can be passed on to the driver - // there are two cases: - // 1. absolute id references (resource_id is set, path is empty) - // 2. relative references (resource_id is set, path starts with a `.`) - if ref.GetResourceId() != nil { - return ref, nil - } - - if !strings.HasPrefix(ref.GetPath(), "/") { - // abort, absolute path references must start with a `/` - return nil, errtypes.BadRequest("ref is invalid: " + ref.String()) - } - - // TODO move mount path trimming to the gateway - fn, err := s.trimMountPrefix(ref.GetPath()) - if err != nil { - return nil, err - } - return &provider.Reference{Path: fn}, nil -} - -func (s *service) trimMountPrefix(fn string) (string, error) { - if strings.HasPrefix(fn, s.mountPath) { - return path.Join("/", strings.TrimPrefix(fn, s.mountPath)), nil - } - return "", errtypes.BadRequest(fmt.Sprintf("path=%q does not belong to this storage provider mount path=%q", fn, s.mountPath)) -} - -func (s *service) wrap(ctx context.Context, ri *provider.ResourceInfo, prefixMountpoint bool) error { - if ri.Id.StorageId == "" { - // For wrapper drivers, the storage ID might already be set. In that case, skip setting it - ri.Id.StorageId = s.mountID - } - if prefixMountpoint { - // TODO move mount path prefixing to the gateway - ri.Path = path.Join(s.mountPath, ri.Path) - } - return nil -} - type descendingMtime []*provider.FileVersion func (v descendingMtime) Len() int { diff --git a/pkg/auth/scope/publicshare.go b/pkg/auth/scope/publicshare.go index 56f893eed0..5bc432087a 100644 --- a/pkg/auth/scope/publicshare.go +++ b/pkg/auth/scope/publicshare.go @@ -94,7 +94,7 @@ func checkStorageRef(ctx context.Context, s *link.PublicShare, r *provider.Refer } // r: - if strings.HasPrefix(r.GetPath(), "/public/"+s.Token) { + if strings.HasPrefix(r.GetPath(), "/public/"+s.Token) || strings.HasPrefix(r.GetPath(), "/"+s.Token) { return true } return false diff --git a/pkg/storage/registry/static/static.go b/pkg/storage/registry/static/static.go index d5d84403e4..394adff176 100644 --- a/pkg/storage/registry/static/static.go +++ b/pkg/storage/registry/static/static.go @@ -46,6 +46,8 @@ var bracketRegex = regexp.MustCompile(`\[(.*?)\]`) type rule struct { Mapping string `mapstructure:"mapping"` Address string `mapstructure:"address"` + ProviderID string `mapstructure:"provider_id"` + ProviderPath string `mapstructure:"provider_path"` Aliases map[string]string `mapstructure:"aliases"` AllowedUserAgents []string `mapstructure:"allowed_user_agents"` } @@ -178,8 +180,9 @@ func (b *reg) FindProviders(ctx context.Context, ref *provider.Reference) ([]*re // TODO(labkode): fill path info based on provider id, if path and storage id points to same id, take that. if m := r.FindString(ref.ResourceId.StorageId); m != "" { return []*registrypb.ProviderInfo{{ - ProviderId: ref.ResourceId.StorageId, - Address: addr, + ProviderId: ref.ResourceId.StorageId, + Address: addr, + ProviderPath: rule.ProviderPath, }}, nil } } @@ -223,6 +226,7 @@ func (b *reg) FindProviders(ctx context.Context, ref *provider.Reference) ([]*re continue } match = ®istrypb.ProviderInfo{ + ProviderId: rule.ProviderID, ProviderPath: m, Address: addr, } @@ -232,6 +236,7 @@ func (b *reg) FindProviders(ctx context.Context, ref *provider.Reference) ([]*re combs := generateRegexCombinations(prefix) for _, c := range combs { shardedMatches = append(shardedMatches, ®istrypb.ProviderInfo{ + ProviderId: rule.ProviderID, ProviderPath: c, Address: addr, }) diff --git a/tests/oc-integration-tests/drone/gateway.toml b/tests/oc-integration-tests/drone/gateway.toml index 916ad90e01..1bbe78ff7e 100644 --- a/tests/oc-integration-tests/drone/gateway.toml +++ b/tests/oc-integration-tests/drone/gateway.toml @@ -64,11 +64,11 @@ home_provider = "/home" # mount a home storage provider that uses a context based path wrapper # to jail users into their home dir -"/home" = {"address" = "localhost:12000"} +"/home" = {"address" = "localhost:12000", "provider_id" = "123e4567-e89b-12d3-a456-426655440000"} # mount a storage provider without a path wrapper for direct access to users. -"/users" = {"address" = "localhost:11000"} -"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:11000"} +"/users" = {"address" = "localhost:11000", "provider_id" = "123e4567-e89b-12d3-a456-426655440000"} +"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:11000", "provider_id" = "123e4567-e89b-12d3-a456-426655440000", "provider_path" = "/users"} # another mount point might be "/projects/" "/public" = {"address" = "localhost:13000"} diff --git a/tests/oc-integration-tests/local/gateway.toml b/tests/oc-integration-tests/local/gateway.toml index 71289e9308..dda9341992 100644 --- a/tests/oc-integration-tests/local/gateway.toml +++ b/tests/oc-integration-tests/local/gateway.toml @@ -57,11 +57,11 @@ home_provider = "/home" # mount a home storage provider that uses a context based path wrapper # to jail users into their home dir -"/home" = {"address" = "localhost:12000"} +"/home" = {"address" = "localhost:12000", "provider_id" = "123e4567-e89b-12d3-a456-426655440000"} # mount a storage provider without a path wrapper for direct access to users. -"/users" = {"address" = "localhost:11000"} -"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:11000"} +"/users" = {"address" = "localhost:11000","provider_id" = "123e4567-e89b-12d3-a456-426655440000"} +"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:11000", "provider_id" = "123e4567-e89b-12d3-a456-426655440000", "provider_path" = "/users"} # another mount point might be "/projects/" "/public" = {"address" = "localhost:13000"}