From e87f673d38cb7ff6219d0c8b10f164e60c368b01 Mon Sep 17 00:00:00 2001 From: Vasco Guita Date: Tue, 7 Mar 2023 16:17:17 +0100 Subject: [PATCH] Temporary fix --- .drone.env | 4 + .drone.star | 242 ++++++++++++++++++ .github/workflows/test.yml | 4 +- .../drone/frontend-global.toml | 114 +++++++++ .../oc-integration-tests/drone/frontend.toml | 125 +++++++++ tests/oc-integration-tests/drone/gateway.toml | 83 ++++++ .../drone/ldap-users.toml | 71 +++++ tests/oc-integration-tests/drone/ocmd.toml | 30 +++ .../drone/providers.demo.json | 100 ++++++++ tests/oc-integration-tests/drone/shares.toml | 20 ++ .../drone/storage-home-ocis.toml | 47 ++++ .../drone/storage-home-s3ng.toml | 55 ++++ .../drone/storage-local-1.toml | 46 ++++ .../drone/storage-local-2.toml | 46 ++++ .../drone/storage-publiclink.toml | 17 ++ .../drone/storage-users-ocis.toml | 42 +++ .../drone/storage-users-s3ng.toml | 49 ++++ .../drone/users.demo.json | 38 +++ tests/oc-integration-tests/drone/users.toml | 21 ++ 19 files changed, 1152 insertions(+), 2 deletions(-) create mode 100644 .drone.env create mode 100644 .drone.star create mode 100644 tests/oc-integration-tests/drone/frontend-global.toml create mode 100644 tests/oc-integration-tests/drone/frontend.toml create mode 100644 tests/oc-integration-tests/drone/gateway.toml create mode 100644 tests/oc-integration-tests/drone/ldap-users.toml create mode 100644 tests/oc-integration-tests/drone/ocmd.toml create mode 100644 tests/oc-integration-tests/drone/providers.demo.json create mode 100644 tests/oc-integration-tests/drone/shares.toml create mode 100644 tests/oc-integration-tests/drone/storage-home-ocis.toml create mode 100644 tests/oc-integration-tests/drone/storage-home-s3ng.toml create mode 100644 tests/oc-integration-tests/drone/storage-local-1.toml create mode 100644 tests/oc-integration-tests/drone/storage-local-2.toml create mode 100644 tests/oc-integration-tests/drone/storage-publiclink.toml create mode 100644 tests/oc-integration-tests/drone/storage-users-ocis.toml create mode 100644 tests/oc-integration-tests/drone/storage-users-s3ng.toml create mode 100644 tests/oc-integration-tests/drone/users.demo.json create mode 100644 tests/oc-integration-tests/drone/users.toml diff --git a/.drone.env b/.drone.env new file mode 100644 index 0000000000..5dd4ad330f --- /dev/null +++ b/.drone.env @@ -0,0 +1,4 @@ +# The test runner source for API tests +APITESTS_COMMITID=804d1777051c3583dfeebae467466f1316fa3583 +APITESTS_BRANCH=master +APITESTS_REPO_GIT_URL=https://github.com/owncloud/ocis.git diff --git a/.drone.star b/.drone.star new file mode 100644 index 0000000000..c55ea1327e --- /dev/null +++ b/.drone.star @@ -0,0 +1,242 @@ +OC_CI_GOLANG = "owncloudci/golang:1.19" +OC_CI_ALPINE = "owncloudci/alpine:latest" +OSIXIA_OPEN_LDAP = "osixia/openldap:1.3.0" +OC_CI_PHP = "cs3org/behat:latest" +OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier:latest" + +def makeStep(): + return { + "name": "build", + "image": OC_CI_GOLANG, + "commands": [ + "make revad", + ], + } + +def cloneApiTestReposStep(): + return { + "name": "clone-api-test-repos", + "image": OC_CI_ALPINE, + "commands": [ + "source /drone/src/.drone.env", + "git clone -b master --depth=1 https://github.com/owncloud/testing.git /drone/src/tmp/testing", + "git clone -b $APITESTS_BRANCH --single-branch --no-tags $APITESTS_REPO_GIT_URL /drone/src/tmp/testrunner", + "cd /drone/src/tmp/testrunner", + "git checkout $APITESTS_COMMITID", + ], + } + +# Shared service definitions +def ldapService(): + return { + "name": "ldap", + "image": OSIXIA_OPEN_LDAP, + "pull": "always", + "environment": { + "LDAP_DOMAIN": "owncloud.com", + "LDAP_ORGANISATION": "ownCloud", + "LDAP_ADMIN_PASSWORD": "admin", + "LDAP_TLS_VERIFY_CLIENT": "never", + "HOSTNAME": "ldap", + }, + } + +def cephService(): + return { + "name": "ceph", + "image": "ceph/daemon", + "pull": "always", + "environment": { + "CEPH_DAEMON": "demo", + "NETWORK_AUTO_DETECT": "4", + "MON_IP": "0.0.0.0", + "CEPH_PUBLIC_NETWORK": "0.0.0.0/0", + "RGW_CIVETWEB_PORT": "4000 ", + "RGW_NAME": "ceph", + "CEPH_DEMO_UID": "test-user", + "CEPH_DEMO_ACCESS_KEY": "test", + "CEPH_DEMO_SECRET_KEY": "test", + "CEPH_DEMO_BUCKET": "test", + }, + } + +# Pipeline definitions +def main(ctx): + return [ + checkStarlark(), + ocisIntegrationTest(), + ] + s3ngIntegrationTests() + +def ocisIntegrationTest(): + return { + "kind": "pipeline", + "type": "docker", + "name": "ocis-integration-tests-2", + "platform": { + "os": "linux", + "arch": "amd64", + }, + "trigger": { + "event": { + "include": [ + "pull_request", + "tag", + ], + }, + }, + "steps": [ + makeStep(), + { + "name": "revad-services", + "image": OC_CI_GOLANG, + "detach": True, + "commands": [ + "cd /drone/src/tests/oc-integration-tests/drone/", + "/drone/src/cmd/revad/revad -c frontend.toml &", + "/drone/src/cmd/revad/revad -c gateway.toml &", + "/drone/src/cmd/revad/revad -c shares.toml &", + "/drone/src/cmd/revad/revad -c storage-home-ocis.toml &", + "/drone/src/cmd/revad/revad -c storage-users-ocis.toml &", + "/drone/src/cmd/revad/revad -c storage-publiclink.toml &", + "/drone/src/cmd/revad/revad -c ldap-users.toml", + ], + }, + cloneApiTestReposStep(), + { + "name": "APIAcceptanceTestsOcisStorage", + "image": OC_CI_PHP, + "commands": [ + "cd /drone/src/tmp/testrunner", + "make test-acceptance-from-core-api", + ], + "environment": { + "TEST_SERVER_URL": "http://revad-services:20080", + "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", + "STORAGE_DRIVER": "OCIS", + "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", + "TEST_WITH_LDAP": "true", + "REVA_LDAP_HOSTNAME": "ldap", + "TEST_REVA": "true", + "SEND_SCENARIO_LINE_REFERENCES": "true", + "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva", + "DIVIDE_INTO_NUM_PARTS": 6, + "RUN_PART": 2, + "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-OCIS-storage.md", + }, + }, + ], + "services": [ + ldapService(), + ], + } + +def s3ngIntegrationTests(): + parallelRuns = 12 + pipelines = [] + for runPart in range(1, parallelRuns + 1): + if runPart in [9]: + continue + + pipelines.append( + { + "kind": "pipeline", + "type": "docker", + "name": "s3ng-integration-tests-%s" % runPart, + "platform": { + "os": "linux", + "arch": "amd64", + }, + "trigger": { + "event": { + "include": [ + "pull_request", + "tag", + ], + }, + }, + "steps": [ + makeStep(), + { + "name": "revad-services", + "image": OC_CI_GOLANG, + "detach": True, + "commands": [ + "cd /drone/src/tests/oc-integration-tests/drone/", + "/drone/src/cmd/revad/revad -c frontend.toml &", + "/drone/src/cmd/revad/revad -c gateway.toml &", + "/drone/src/cmd/revad/revad -c shares.toml &", + "/drone/src/cmd/revad/revad -c storage-home-s3ng.toml &", + "/drone/src/cmd/revad/revad -c storage-users-s3ng.toml &", + "/drone/src/cmd/revad/revad -c storage-publiclink.toml &", + "/drone/src/cmd/revad/revad -c ldap-users.toml", + ], + }, + cloneApiTestReposStep(), + { + "name": "APIAcceptanceTestsS3ngStorage", + "image": OC_CI_PHP, + "commands": [ + "cd /drone/src/tmp/testrunner", + "make test-acceptance-from-core-api", + ], + "environment": { + "TEST_SERVER_URL": "http://revad-services:20080", + "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/", + "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/nodes/root/* /drone/src/tmp/reva/data/nodes/*-*-*-* /drone/src/tmp/reva/data/blobs/*", + "STORAGE_DRIVER": "S3NG", + "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton", + "TEST_WITH_LDAP": "true", + "REVA_LDAP_HOSTNAME": "ldap", + "TEST_REVA": "true", + "SEND_SCENARIO_LINE_REFERENCES": "true", + "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@personalSpace&&~@issue-ocis-3023&&~&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva", + "DIVIDE_INTO_NUM_PARTS": parallelRuns, + "RUN_PART": runPart, + "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-S3NG-storage.md", + }, + }, + ], + "services": [ + ldapService(), + cephService(), + ], + }, + ) + + return pipelines + +def checkStarlark(): + return { + "kind": "pipeline", + "type": "docker", + "name": "check-starlark", + "steps": [ + { + "name": "format-check-starlark", + "image": OC_CI_BAZEL_BUILDIFIER, + "commands": [ + "buildifier --mode=check .drone.star", + ], + }, + { + "name": "show-diff", + "image": OC_CI_BAZEL_BUILDIFIER, + "commands": [ + "buildifier --mode=fix .drone.star", + "git diff", + ], + "when": { + "status": [ + "failure", + ], + }, + }, + ], + "depends_on": [], + "trigger": { + "ref": [ + "refs/pull/**", + ], + }, + } diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb636d57a9..2bd4564c66 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,7 +78,7 @@ jobs: strategy: fail-fast: false matrix: - part: [1, 2, 3, 4, 5, 6] + part: [1, 3, 4, 5, 6] uses: ./.github/workflows/compose.yml with: test: acceptance-2 @@ -90,7 +90,7 @@ jobs: strategy: fail-fast: false matrix: - part: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + part: [9] uses: ./.github/workflows/compose.yml with: test: acceptance-3 diff --git a/tests/oc-integration-tests/drone/frontend-global.toml b/tests/oc-integration-tests/drone/frontend-global.toml new file mode 100644 index 0000000000..12d5a2bc92 --- /dev/null +++ b/tests/oc-integration-tests/drone/frontend-global.toml @@ -0,0 +1,114 @@ +# This config file will start a reva service that: +# - serves as the entrypoint for owncloud APIs but with a globally accessible namespace. +# - serves http endpoints on port 20180 +# - / --------------- ocdav +# - /ocs ------------ ocs +# - TODO(diocas): ocm + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +[http] +address = "0.0.0.0:20180" + +[http.middlewares.cors] +allow_credentials = true + +[http.services.ocdav] +# serve ocdav on the root path +prefix = "" +chunk_folder = "/drone/src/tmp/reva/chunks" +# for user lookups +# prefix the path of requests to /dav/files with this namespace +# While owncloud has only listed usernames at this endpoint CERN has +# been exposing more than just usernames. For owncloud deployments we +# can prefix the path to jail the requests to the correct CS3 namespace. +# In this deployment we mounted the owncloud storage provider at /users. It +# expects a username as the first path segment. +# currently, only the desktop client will use this endpoint, but only if +# the dav.chunking capability is available +# TODO implement a path wrapper that rewrites `` into the path +# layout for the users home? +# no, use GetHome? +# for eos we need to rewrite the path +# TODO strip the username from the path so the CS3 namespace can be mounted +# at the files/ endpoint? what about migration? separate reva instance +files_namespace = "/" + +# similar to the dav/files endpoint we can configure a prefix for the old webdav endpoint +# we use the old webdav endpoint to present the cs3 namespace +# note: this changes the tree that is rendered at remote.php/webdav from the users home to the cs3 namespace +# use webdav_namespace = "/home" to use the old namespace that only exposes the users files +# this endpoint should not affect the desktop client sync but will present different folders for the other clients: +# - the desktop clients use a hardcoded remote.php/dav/files/ if the dav.chunkung capability is present +# - the ios ios uses the core.webdav-root capability which points to remote.php/webdav in oc10 +# - the oc js sdk is hardcoded to the remote.php/webdav so it will see the new tree +# - TODO android? no sync ... but will see different tree +webdav_namespace = "/" + +[http.services.ocs] + +[http.services.ocs.capabilities.capabilities.core.status] +version = "10.0.11.5" +versionstring = "10.0.11" + +[http.services.ocs.capabilities.capabilities.files_sharing] +api_enabled = true +resharing = true +group_sharing = true +auto_accept_share = true +share_with_group_members_only = true +share_with_membership_groups_only = true +default_permissions = 22 +search_min_length = 3 + +[http.services.ocs.capabilities.capabilities.files_sharing.public] +enabled = true +send_mail = true +social_share = true +upload = true +multiple = true +supports_upload_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.password] +enforced = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.password.enforced_for] +read_only = true +read_write = true +upload_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.expire_date] +enabled = true + +[http.services.ocs.capabilities.capabilities.files_sharing.user] +send_mail = true + +[http.services.ocs.capabilities.capabilities.files_sharing.user_enumeration] +enabled = true +group_members_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.federation] +outgoing = true +incoming = true + +[http.services.ocs.capabilities.capabilities.notifications] +endpoints = [] + +[http.services.ocs.capabilities.capabilities.files.tus_support] +version = "1.0.0" +resumable = "1.0.0" +extension = "creation,creation-with-upload" +http_method_override = "" +max_chunk_size = 0 + +# serve /ocm +[http.services.ocmd] +prefix = "ocm" + +[http.middlewares.providerauthorizer] +driver = "json" + +[http.middlewares.providerauthorizer.drivers.json] +providers = "providers.demo.json" \ No newline at end of file diff --git a/tests/oc-integration-tests/drone/frontend.toml b/tests/oc-integration-tests/drone/frontend.toml new file mode 100644 index 0000000000..84ed3b43f7 --- /dev/null +++ b/tests/oc-integration-tests/drone/frontend.toml @@ -0,0 +1,125 @@ +# This config file will start a reva service that: +# - serves as the entrypoint for owncloud APIs. +# - jails users into their home folder as in owncloud classic +# - serves http endpoints on port 20080 +# - / --------------- ocdav +# - /ocs ------------ ocs +# - TODO(diocas): ocm + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +[grpc] +address = "0.0.0.0:20099" + +[grpc.services.authprovider] +auth_manager = "oidc" + +# If you want to use your own openid provider change this config +[grpc.services.authprovider.auth_managers.oidc] +issuer = "http://localhost:20080" + +[http] +address = "0.0.0.0:20080" + +[http.middlewares.cors] +allow_credentials = true + +[http.services.ocdav] +# serve ocdav on the root path +prefix = "" +chunk_folder = "/drone/src/tmp/reva/chunks" +# for user lookups +# prefix the path of requests to /dav/files with this namespace +# While owncloud has only listed usernames at this endpoint CERN has +# been exposing more than just usernames. For owncloud deployments we +# can prefix the path to jail the requests to the correct CS3 namespace. +# In this deployment we mounted the owncloud storage provider at /users. It +# expects a username as the first path segment. +# currently, only the desktop client will use this endpoint, but only if +# the dav.chunking capability is available +# TODO implement a path wrapper that rewrites `` into the path +# layout for the users home? +# no, use GetHome? +# for eos we need to rewrite the path +# TODO strip the username from the path so the CS3 namespace can be mounted +# at the files/ endpoint? what about migration? separate reva instance +files_namespace = "/users" + +# similar to the dav/files endpoint we can configure a prefix for the old webdav endpoint +# we use the old webdav endpoint to present the cs3 namespace +# note: this changes the tree that is rendered at remote.php/webdav from the users home to the cs3 namespace +# use webdav_namespace = "/home" to use the old namespace that only exposes the users files +# this endpoint should not affect the desktop client sync but will present different folders for the other clients: +# - the desktop clients use a hardcoded remote.php/dav/files/ if the dav.chunkung capability is present +# - the ios ios uses the core.webdav-root capability which points to remote.php/webdav in oc10 +# - the oc js sdk is hardcoded to the remote.php/webdav so it will see the new tree +# - TODO android? no sync ... but will see different tree +webdav_namespace = "/home" + +[http.services.ocs] + +[http.services.ocs.capabilities.capabilities.core.status] +version = "10.0.11.5" +versionstring = "10.0.11" + +[http.services.ocs.capabilities.capabilities.files_sharing] +api_enabled = true +resharing = true +group_sharing = true +auto_accept_share = true +share_with_group_members_only = true +share_with_membership_groups_only = true +default_permissions = 22 +search_min_length = 3 + +[http.services.ocs.capabilities.capabilities.files_sharing.public] +enabled = true +send_mail = true +social_share = true +upload = true +multiple = true +supports_upload_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.password] +enforced = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.password.enforced_for] +read_only = true +read_write = true +upload_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.public.expire_date] +enabled = true + +[http.services.ocs.capabilities.capabilities.files_sharing.user] +send_mail = true + +[http.services.ocs.capabilities.capabilities.files_sharing.user_enumeration] +enabled = true +group_members_only = true + +[http.services.ocs.capabilities.capabilities.files_sharing.federation] +outgoing = true +incoming = true + +[http.services.ocs.capabilities.capabilities.notifications] +endpoints = [] + +[http.services.ocs.capabilities.capabilities.files.tus_support] +version = "1.0.0" +resumable = "1.0.0" +extension = "creation,creation-with-upload" +http_method_override = "" +max_chunk_size = 0 + +# serve /ocm +[http.services.ocmd] +prefix = "ocm" + +[http.middlewares.providerauthorizer] +driver = "json" + +[http.middlewares.providerauthorizer.drivers.json] +providers = "providers.demo.json" \ No newline at end of file diff --git a/tests/oc-integration-tests/drone/gateway.toml b/tests/oc-integration-tests/drone/gateway.toml new file mode 100644 index 0000000000..85c17b20c2 --- /dev/null +++ b/tests/oc-integration-tests/drone/gateway.toml @@ -0,0 +1,83 @@ +# This config file will start a reva service that: +# - serves as a gateway for all CS3 requests +# - looks up the storageprovider using a storageregistry +# - looks up the authprovider using an authregistry +# - serves the gateway on grpc port 19000 +# - serves http datagateway on port 19001 +# - /data - datagateway: file up and download + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +[grpc] +address = "0.0.0.0:19000" + +[grpc.services.gateway] +# registries +authregistrysvc = "localhost:19000" +storageregistrysvc = "localhost:19000" +# user metadata +preferencessvc = "localhost:18000" +userprovidersvc = "localhost:18000" +groupprovidersvc = "localhost:18000" +# an approvider lives on "localhost:18000" as well, see users.toml +# sharing +usershareprovidersvc = "localhost:17000" +publicshareprovidersvc = "localhost:17000" +# ocm +ocmcoresvc = "localhost:14000" +ocmshareprovidersvc = "localhost:14000" +ocminvitemanagersvc = "localhost:14000" +ocmproviderauthorizersvc = "localhost:14000" +# permissions +permissionssvc = "localhost:10000" +# other +commit_share_to_storage_grant = true +commit_share_to_storage_ref = true +share_folder = "Shares" +datagateway = "http://localhost:19001/data" +transfer_shared_secret = "replace-me-with-a-transfer-secret" # for direct uploads +transfer_expires = 6 # give it a moment +#disable_home_creation_on_login = true +link_grants_file = "/drone/src/tmp/reva/link_grants_file.json" + +[grpc.services.authregistry] +driver = "static" + +[grpc.services.authregistry.drivers.static.rules] +publicshares = "localhost:17000" # started with the shares.toml +basic = "localhost:18000" # started with the users.toml + +[grpc.services.storageregistry] +driver = "static" + +[grpc.services.storageregistry.drivers.static] +home_provider = "/home" + +[grpc.services.storageregistry.drivers.static.rules] + +# mount a home storage provider that uses a context based path wrapper +# to jail users into their home dir +"/home" = {"address" = "localhost:12000"} + +# mount a storage provider without a path wrapper for direct access to users. +"/users" = {"address" = "localhost:11000"} +"123e4567-e89b-12d3-a456-426655440000" = {"address" = "localhost:11000"} + +# the /virtual namespace is only accessible via the frontend-global service +"/virtual/[a-k]" = {"address" = "localhost:11100"} +"virtual-a-k" = {"address" = "localhost:11100"} +"/virtual/[l-z]" = {"address" = "localhost:11110"} +"virtual-l-z" = {"address" = "localhost:11110"} + +# another mount point might be "/projects/" + +"/public" = {"address" = "localhost:13000"} +"e1a73ede-549b-4226-abdf-40e69ca8230d" = {"address" = "localhost:13000"} + +[http] +address = "0.0.0.0:19001" + +[http.services.datagateway] +transfer_shared_secret = "replace-me-with-a-transfer-secret" diff --git a/tests/oc-integration-tests/drone/ldap-users.toml b/tests/oc-integration-tests/drone/ldap-users.toml new file mode 100644 index 0000000000..0cd4f47dab --- /dev/null +++ b/tests/oc-integration-tests/drone/ldap-users.toml @@ -0,0 +1,71 @@ +# This config file will start a reva service that: +# - handles user metadata and user preferences +# - serves the grpc services on port 18000 + +[shared] +jwt_secret = "Pive-Fumkiu4" + +[grpc] +address = "0.0.0.0:18000" + +[grpc.services.authprovider] +auth_manager = "ldap" + +[grpc.services.authprovider.auth_managers.ldap] +hostname="ldap" +port=636 +insecure=true +base_dn="dc=owncloud,dc=com" +loginfilter="(&(objectclass=posixAccount)(|(cn={{login}}))(uid={{login}}))" +bind_username="cn=admin,dc=owncloud,dc=com" +bind_password="admin" +idp="http://localhost:18000" +[grpc.services.authprovider.auth_managers.ldap.schema] +uid="uid" +displayName="displayName" +dn="dn" +cn="cn" + +[grpc.services.userprovider] +driver = "ldap" + +[grpc.services.userprovider.drivers.ldap] +hostname="ldap" +port=636 +insecure=true +base_dn="dc=owncloud,dc=com" +userfilter="(&(objectclass=posixAccount)(|(uid={{.OpaqueId}})(cn={{.OpaqueId}})))" +findfilter="(&(objectclass=posixAccount)(|(cn={{query}}*)(displayname={{query}}*)(mail={{query}}*)))" +attributefilter="(&(objectclass=posixAccount)({{attr}}={{value}}))" +groupfilter="(&(objectclass=posixGroup)(cn=*)(memberuid={{.OpaqueId}}))" +bind_username="cn=admin,dc=owncloud,dc=com" +bind_password="admin" +idp="http://localhost:18000" + +[grpc.services.userprovider.drivers.ldap.schema] +uid="uid" +displayName="displayName" +dn="dn" +cn="cn" + +[grpc.services.groupprovider] +driver = "ldap" + +[grpc.services.groupprovider.drivers.ldap] +hostname="ldap" +port=636 +insecure=true +base_dn="dc=owncloud,dc=com" +groupfilter="(&(objectclass=posixGroup)(|(gid={{.OpaqueId}})(cn={{.OpaqueId}})))" +findfilter="(&(objectclass=posixGroup)(|(cn={{query}}*)(displayname={{query}}*)(mail={{query}}*)))" +attributefilter="(&(objectclass=posixGroup)({{attr}}={{value}}))" +memberfilter="(&(objectclass=posixAccount)(cn={{.OpaqueId}}))" +bind_username="cn=admin,dc=owncloud,dc=com" +bind_password="admin" +idp="http://localhost:18000" + +[grpc.services.groupprovider.drivers.ldap.schema] +gid="cn" +displayName="cn" +dn="dn" +cn="cn" diff --git a/tests/oc-integration-tests/drone/ocmd.toml b/tests/oc-integration-tests/drone/ocmd.toml new file mode 100644 index 0000000000..0439d83856 --- /dev/null +++ b/tests/oc-integration-tests/drone/ocmd.toml @@ -0,0 +1,30 @@ +# This config file will start a reva service that: +# - serves user and public link shares + +[shared] +jwt_secret = "Pive-Fumkiu4" + +[grpc] +address = "0.0.0.0:14000" + +[grpc.services.ocmcore] +driver = "json" + +# Note that ocmcore and ocmshareprovider should use the same file for storing the shares. +[grpc.services.ocmcore.drivers.json] +file = "/drone/src/var/tmp/reva/shares_server_1.json" + +[grpc.services.ocminvitemanager] +driver = "json" + +[grpc.services.ocmshareprovider] +driver = "json" + +[grpc.services.ocmshareprovider.drivers.json] +file = "/drone/src/var/tmp/reva/shares_server_1.json" + +[grpc.services.ocmproviderauthorizer] +driver = "json" + +[grpc.services.ocmproviderauthorizer.drivers.json] +providers = "providers.demo.json" diff --git a/tests/oc-integration-tests/drone/providers.demo.json b/tests/oc-integration-tests/drone/providers.demo.json new file mode 100644 index 0000000000..f2111b71b2 --- /dev/null +++ b/tests/oc-integration-tests/drone/providers.demo.json @@ -0,0 +1,100 @@ +[ + { + "name": "local", + "full_name": "ownCloud@Example", + "organization": "Example", + "domain": "example.org", + "homepage": "http://example.org", + "description": "Example cloud storage.", + "services": [ + { + "endpoint": { + "type": { + "name": "OCM", + "description": "Example Open Cloud Mesh API" + }, + "name": "Example - OCM API", + "path": "http://127.0.0.1:20080/ocm/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "127.0.0.1:20080" + }, + { + "endpoint": { + "type": { + "name": "Webdav", + "description": "Example Webdav API" + }, + "name": "Example - Webdav API", + "path": "http://127.0.0.1:20080/remote.php/webdav/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "127.0.0.1:20080" + }, + { + "endpoint": { + "type": { + "name": "Gateway", + "description": "Example GRPC Gateway" + }, + "name": "Example - GRPC Gateway", + "path": "127.0.0.1:19000", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "127.0.0.1:19000" + } + ] + }, + { + "name": "local-mesh", + "full_name": "ownCloud@Example2", + "organization": "Example 2", + "domain": "example.org", + "homepage": "http://example.org", + "description": "Example cloud storage 2.", + "services": [ + { + "endpoint": { + "type": { + "name": "OCM", + "description": "Example 2 Open Cloud Mesh API" + }, + "name": "Example 2 - OCM API", + "path": "http://127.0.0.1:40080/ocm/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "127.0.0.1:40080" + }, + { + "endpoint": { + "type": { + "name": "Webdav", + "description": "Example 2 Webdav API" + }, + "name": "Example 2 - Webdav API", + "path": "http://127.0.0.1:40080/remote.php/webdav/", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "127.0.0.1:40080" + }, + { + "endpoint": { + "type": { + "name": "Gateway", + "description": "Example 2 GRPC Gateway" + }, + "name": "Example 2 - GRPC Gateway", + "path": "127.0.0.1:39000", + "is_monitored": true + }, + "api_version": "0.0.1", + "host": "127.0.0.1:39000" + } + ] + } +] diff --git a/tests/oc-integration-tests/drone/shares.toml b/tests/oc-integration-tests/drone/shares.toml new file mode 100644 index 0000000000..34f5cf2c16 --- /dev/null +++ b/tests/oc-integration-tests/drone/shares.toml @@ -0,0 +1,20 @@ +# This config file will start a reva service that: +# - serves user and public link shares + +[shared] +jwt_secret = "Pive-Fumkiu4" + +[grpc] +address = "0.0.0.0:17000" + +[grpc.services.usershareprovider] +driver = "memory" + +[grpc.services.authprovider] +auth_manager = "publicshares" + +[grpc.services.authprovider.auth_managers.publicshares] +gateway_addr = "0.0.0.0:19000" + +[grpc.services.publicshareprovider.drivers.json] +file = "/drone/src/tmp/reva/publicshares.json" diff --git a/tests/oc-integration-tests/drone/storage-home-ocis.toml b/tests/oc-integration-tests/drone/storage-home-ocis.toml new file mode 100644 index 0000000000..098aaf7fca --- /dev/null +++ b/tests/oc-integration-tests/drone/storage-home-ocis.toml @@ -0,0 +1,47 @@ +# This config file will start a reva service that: +# - uses the ocis driver to serve users, jailed into their home (/home) +# - serves the home storage provider on grpc port 12000 +# - serves http dataprovider for this storage on port 12001 +# - /data - dataprovider: file up and download +# +# The home storage will inject the username into the path and jail users into +# their home directory + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +[grpc] +address = "0.0.0.0:12000" + +# This is a storage provider that grants direct access to the wrapped storage +# the context path wrapper reads tho username from the context and prefixes the relative storage path with it +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/home" +mount_id = "123e4567-e89b-12d3-a456-426655440000" +expose_data_server = true +data_server_url = "http://revad-services:12001/data" +enable_home_creation = true +gateway_addr = "0.0.0.0:19000" + +[grpc.services.storageprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data" +enable_home = true +treetime_accounting = true +treesize_accounting = true +gateway_addr = "0.0.0.0:19000" + +# we have a locally running dataprovider +[http] +address = "0.0.0.0:12001" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/drone/src/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data" +enable_home = true +treetime_accounting = true +treesize_accounting = true diff --git a/tests/oc-integration-tests/drone/storage-home-s3ng.toml b/tests/oc-integration-tests/drone/storage-home-s3ng.toml new file mode 100644 index 0000000000..dfd784b567 --- /dev/null +++ b/tests/oc-integration-tests/drone/storage-home-s3ng.toml @@ -0,0 +1,55 @@ +# This config file will start a reva service that: +# - uses the s3ng driver to serve users, jailed into their home (/home) +# - serves the home storage provider on grpc port 12000 +# - serves http dataprovider for this storage on port 12001 +# - /data - dataprovider: file up and download +# +# The home storage will inject the username into the path and jail users into +# their home directory + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +[grpc] +address = "0.0.0.0:12000" + +# This is a storage provider that grants direct access to the wrapped storage +# the context path wrapper reads tho username from the context and prefixes the relative storage path with it +[grpc.services.storageprovider] +driver = "s3ng" +mount_path = "/home" +mount_id = "123e4567-e89b-12d3-a456-426655440000" +expose_data_server = true +data_server_url = "http://revad-services:12001/data" +enable_home_creation = true + +[grpc.services.storageprovider.drivers.s3ng] +root = "/drone/src/tmp/reva/data" +enable_home = true +treetime_accounting = true +treesize_accounting = true +"s3.endpoint" = "http://ceph:8080" +"s3.region" = "default" +"s3.bucket" = "test" +"s3.access_key" = "test" +"s3.secret_key" = "test" + +# we have a locally running dataprovider +[http] +address = "0.0.0.0:12001" + +[http.services.dataprovider] +driver = "s3ng" +temp_folder = "/drone/src/tmp/reva/tmp" + +[http.services.dataprovider.drivers.s3ng] +root = "/drone/src/tmp/reva/data" +enable_home = true +treetime_accounting = true +treesize_accounting = true +"s3.endpoint" = "http://ceph:8080" +"s3.region" = "default" +"s3.bucket" = "test" +"s3.access_key" = "test" +"s3.secret_key" = "test" diff --git a/tests/oc-integration-tests/drone/storage-local-1.toml b/tests/oc-integration-tests/drone/storage-local-1.toml new file mode 100644 index 0000000000..3b7d166d87 --- /dev/null +++ b/tests/oc-integration-tests/drone/storage-local-1.toml @@ -0,0 +1,46 @@ +# This config file will start a reva service that: +# - uses the ocis driver to serve one half of a virtual view folder (/virtual/[a-k]) +# - serves the storage provider on grpc port 11100 +# - serves http dataprovider for this storage on port 11101 +# - /data - dataprovider: file up and download + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +[grpc] +address = "0.0.0.0:11100" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/virtual" +mount_id = "virtual-a-k" +expose_data_server = true +data_server_url = "http://localhost:11101/data" + +[grpc.services.storageprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data-local-1" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true + +[http] +address = "0.0.0.0:11101" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/drone/src/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data-local-1" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true diff --git a/tests/oc-integration-tests/drone/storage-local-2.toml b/tests/oc-integration-tests/drone/storage-local-2.toml new file mode 100644 index 0000000000..db3e79fe5f --- /dev/null +++ b/tests/oc-integration-tests/drone/storage-local-2.toml @@ -0,0 +1,46 @@ +# This config file will start a reva service that: +# - uses the ocis driver to serve one half of a virtual view folder (/virtual/[l-z]) +# - serves the storage provider on grpc port 11110 +# - serves http dataprovider for this storage on port 11111 +# - /data - dataprovider: file up and download + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +[grpc] +address = "0.0.0.0:11110" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/virtual" +mount_id = "virtual-l-z" +expose_data_server = true +data_server_url = "http://localhost:11111/data" + +[grpc.services.storageprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data-local-2" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true + +[http] +address = "0.0.0.0:11111" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/drone/src/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data-local-2" +owner = "4c510ada-c86b-4815-8820-42cdf82c3d51" +owner_idp = "http://localhost:20080" +owner_type = "USER_TYPE_PRIMARY" +enable_home = false +treetime_accounting = true +treesize_accounting = true diff --git a/tests/oc-integration-tests/drone/storage-publiclink.toml b/tests/oc-integration-tests/drone/storage-publiclink.toml new file mode 100644 index 0000000000..f36aeb18fd --- /dev/null +++ b/tests/oc-integration-tests/drone/storage-publiclink.toml @@ -0,0 +1,17 @@ +# This storage.toml config file will start a reva service that: +# - serves the public storage provider on grpc port 13000 + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +[grpc] +address = "0.0.0.0:13000" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.publicstorageprovider] +mount_path = "/public/" +mount_id = "e1a73ede-549b-4226-abdf-40e69ca8230d" +gateway_addr = "0.0.0.0:19000" + diff --git a/tests/oc-integration-tests/drone/storage-users-ocis.toml b/tests/oc-integration-tests/drone/storage-users-ocis.toml new file mode 100644 index 0000000000..795ba41d54 --- /dev/null +++ b/tests/oc-integration-tests/drone/storage-users-ocis.toml @@ -0,0 +1,42 @@ +# This config file will start a reva service that: +# - uses the ocis driver to serve users (/users) +# - serves the storage provider on grpc port 11000 +# - serves http dataprovider for this storage on port 11001 +# - /data - dataprovider: file up and download + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +[grpc] +address = "0.0.0.0:11000" + +# This is a storage provider that grants direct access to the wrapped storage +# we have a locally running dataprovider +[grpc.services.storageprovider] +driver = "ocis" +mount_path = "/users" +mount_id = "123e4567-e89b-12d3-a456-426655440000" +expose_data_server = true +data_server_url = "http://revad-services:11001/data" +gateway_addr = "0.0.0.0:19000" + +[grpc.services.storageprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data" +treetime_accounting = true +treesize_accounting = true +userprovidersvc = "localhost:18000" +gateway_addr = "0.0.0.0:19000" + +# we have a locally running dataprovider +[http] +address = "0.0.0.0:11001" + +[http.services.dataprovider] +driver = "ocis" +temp_folder = "/drone/src/tmp/reva/tmp" + +[http.services.dataprovider.drivers.ocis] +root = "/drone/src/tmp/reva/data" +treetime_accounting = true +treesize_accounting = true diff --git a/tests/oc-integration-tests/drone/storage-users-s3ng.toml b/tests/oc-integration-tests/drone/storage-users-s3ng.toml new file mode 100644 index 0000000000..63073c9585 --- /dev/null +++ b/tests/oc-integration-tests/drone/storage-users-s3ng.toml @@ -0,0 +1,49 @@ +# This config file will start a reva service that: +# - uses the s3ng driver to serve users (/users) +# - serves the storage provider on grpc port 11000 +# - serves http dataprovider for this storage on port 11001 +# - /data - dataprovider: file up and download + +[shared] +jwt_secret = "Pive-Fumkiu4" +gatewaysvc = "localhost:19000" + +[grpc] +address = "0.0.0.0:11000" + +# This is a storage provider that grants direct access to the wrapped storage +[grpc.services.storageprovider] +driver = "s3ng" +mount_path = "/users" +mount_id = "123e4567-e89b-12d3-a456-426655440000" +expose_data_server = true +data_server_url = "http://revad-services:11001/data" + +[grpc.services.storageprovider.drivers.s3ng] +root = "/drone/src/tmp/reva/data" +treetime_accounting = true +treesize_accounting = true +userprovidersvc = "localhost:18000" +"s3.endpoint" = "http://ceph:8080" +"s3.region" = "default" +"s3.bucket" = "test" +"s3.access_key" = "test" +"s3.secret_key" = "test" + +# we have a locally running dataprovider +[http] +address = "0.0.0.0:11001" + +[http.services.dataprovider] +driver = "s3ng" +temp_folder = "/drone/src/tmp/reva/tmp" + +[http.services.dataprovider.drivers.s3ng] +root = "/drone/src/tmp/reva/data" +treetime_accounting = true +treesize_accounting = true +"s3.endpoint" = "http://ceph:8080" +"s3.region" = "default" +"s3.bucket" = "test" +"s3.access_key" = "test" +"s3.secret_key" = "test" diff --git a/tests/oc-integration-tests/drone/users.demo.json b/tests/oc-integration-tests/drone/users.demo.json new file mode 100644 index 0000000000..893d69b99f --- /dev/null +++ b/tests/oc-integration-tests/drone/users.demo.json @@ -0,0 +1,38 @@ +[ + { + "id": { + "opaque_id": "4c510ada-c86b-4815-8820-42cdf82c3d51", + "idp": "http://localhost:20080", + "type": 1 + }, + "username": "einstein", + "secret": "relativity", + "mail": "einstein@example.org", + "display_name": "Albert Einstein", + "groups": ["sailing-lovers", "violin-haters", "physics-lovers"] + }, + { + "id": { + "opaque_id": "f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c", + "idp": "http://localhost:20080", + "type": 1 + }, + "username": "marie", + "secret": "radioactivity", + "mail": "marie@example.org", + "display_name": "Marie Curie", + "groups": ["radium-lovers", "polonium-lovers", "physics-lovers"] + }, + { + "id": { + "opaque_id": "932b4540-8d16-481e-8ef4-588e4b6b151c", + "idp": "http://localhost:20080", + "type": 1 + }, + "username": "richard", + "secret": "superfluidity", + "mail": "richard@example.org", + "display_name": "Richard Feynman", + "groups": ["quantum-lovers", "philosophy-haters", "physics-lovers"] + } +] diff --git a/tests/oc-integration-tests/drone/users.toml b/tests/oc-integration-tests/drone/users.toml new file mode 100644 index 0000000000..6f849bf6a8 --- /dev/null +++ b/tests/oc-integration-tests/drone/users.toml @@ -0,0 +1,21 @@ +# This config file will start a reva service that: +# - handles user metadata and user preferences +# - serves the grpc services on port 18000 + +[shared] +jwt_secret = "Pive-Fumkiu4" + +[grpc] +address = "0.0.0.0:18000" + +[grpc.services.authprovider] +auth_manager = "json" + +[grpc.services.authprovider.auth_managers.json] +users = "users.demo.json" + +[grpc.services.userprovider] +driver = "json" + +[grpc.services.userprovider.drivers.json] +users = "users.demo.json"