diff --git a/.drone.star b/.drone.star index 3e6d063a700..c263f311b28 100644 --- a/.drone.star +++ b/.drone.star @@ -1413,7 +1413,6 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "STORAGE_HOME_DRIVER": "%s" % (storage), "STORAGE_USERS_DRIVER": "%s" % (storage), "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root", - "STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data", "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users", "STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata", "STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json", @@ -1465,7 +1464,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "STORAGE_LDAP_USERATTRIBUTEFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)({{attr}}={{value}}))", "STORAGE_LDAP_USERFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(ownclouduuid={{.OpaqueId}})(uid={{.OpaqueId}})))", "STORAGE_LDAP_USERFINDFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(cn={{query}}*)(displayname={{query}}*)(mail={{query}}*)))", - # ownCloud storage driver + # ownCloudSQL storage driver "STORAGE_HOME_DRIVER": "owncloudsql", "STORAGE_USERS_DRIVER": "owncloudsql", "STORAGE_METADATA_DRIVER": "ocis", @@ -1480,7 +1479,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBNAME": "owncloud", # TODO: redis is not yet supported "STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR": "redis:6379", - # ownCloud sharing driver + # ownCloudSQL sharing driver "STORAGE_SHARING_USER_DRIVER": "oc10-sql", "STORAGE_SHARING_USER_SQL_USERNAME": "owncloud", "STORAGE_SHARING_USER_SQL_PASSWORD": "owncloud", diff --git a/changelog/unreleased/change-remove-owncloud-storage-driver.md b/changelog/unreleased/change-remove-owncloud-storage-driver.md new file mode 100644 index 00000000000..568265117dc --- /dev/null +++ b/changelog/unreleased/change-remove-owncloud-storage-driver.md @@ -0,0 +1,10 @@ +Change: remove the ownCloud storage driver + +We've removed the ownCloud storage driver because it was no longer +maintained after the ownCloud SQL storage driver was added. + +If you have been using the ownCloud storage driver, please switch +to the ownCloud SQL storage driver which brings you more features and +is under active maintenance. + +https://github.com/owncloud/ocis/pull/3072 diff --git a/deployments/examples/oc10_ocis_parallel/docker-compose.yml b/deployments/examples/oc10_ocis_parallel/docker-compose.yml index bec46cb653b..b489fff4c49 100644 --- a/deployments/examples/oc10_ocis_parallel/docker-compose.yml +++ b/deployments/examples/oc10_ocis_parallel/docker-compose.yml @@ -87,7 +87,7 @@ services: STORAGE_LDAP_USERATTRIBUTEFILTER: "(&(objectclass=posixAccount)(objectclass=owncloud)({{attr}}={{value}}))" STORAGE_LDAP_USERFILTER: "(&(objectclass=posixAccount)(objectclass=owncloud)(|(ownclouduuid={{.OpaqueId}})(uid={{.OpaqueId}})))" STORAGE_LDAP_USERFINDFILTER: "(&(objectclass=posixAccount)(objectclass=owncloud)(|(cn={{query}}*)(displayname={{query}}*)(mail={{query}}*)))" - # ownCloud storage driver + # ownCloudSQL storage driver STORAGE_USERS_DRIVER: owncloudsql STORAGE_METADATA_DRIVER: ocis # keep metadata on ocis storage since this are only small files atm STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR: /mnt/data/files @@ -100,7 +100,7 @@ services: STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPORT: 3306 STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBNAME: owncloud STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR: redis:6379 # TODO: redis is not yet supported - # ownCloud sharing driver + # ownCloudSQL sharing driver STORAGE_SHARING_USER_DRIVER: oc10-sql STORAGE_SHARING_USER_SQL_USERNAME: owncloud STORAGE_SHARING_USER_SQL_PASSWORD: owncloud diff --git a/docs/extensions/storage/storagedrivers.md b/docs/extensions/storage/storagedrivers.md index 70afc19d90b..b879a4e8eae 100644 --- a/docs/extensions/storage/storagedrivers.md +++ b/docs/extensions/storage/storagedrivers.md @@ -90,7 +90,7 @@ The S3NG storage driver uses the same metadata layout on a POSIX storage as the TODO add list of capabilities / tradeoffs -#### Related Kernel limits +#### Related Kernel limits The decomposedfs currently stores CS3 grants in extended attributes. When listing extended attributes the result is currently limited to 64kB. Assuming a 20 byte uuid a grant has ~40 bytes. Which would limit the number of extended attributes to ~1630 entries or ~1600 shares. This can be extended by moving the grants from extended attributes into a dedicated file. From [Wikipedia on Extended file attributes](https://en.wikipedia.org/wiki/Extended_file_attributes#Linux): @@ -134,49 +134,6 @@ The *minimal* storage driver for a POSIX based filesystem. It literally supports To provide the other storage aspects we plan to implement a FUSE overlay filesystem which will add the different aspects on top of local filesystems like ext4, btrfs or xfs. It should work on NFSv45 as well, although NFSv4 supports RichACLs and we will explore how to leverage them to implement sharing at a future date. The idea is to use the storages native capabilities to deliver the best user experience. But again: that means making the right tradeoffs. -### OwnCloud Storage Driver - -{{< hint info >}} -**Deprecated** -The owncloud storage driver was used for testing purposes early on. For a parallel deployment focus has shifted to the owncloudsql. -{{< /hint >}} - -While this storage driver implements the file tree (using redis, including id based lookup), ETag propagation, trash, versions and sharing (including expiry) using the data directory layout of ownCloud 10 it has [known limitations](https://github.com/owncloud/core/issues/28095) that cannot be fixed without changing the actual layout on disk. - -To setup it up properly in a distributed fashion, the storage-home and the storage-oc need to share the same underlying FS. Their "data" counterparts also need access to the same shared FS. -For a simple docker-compose setup, you can create a volume which will be used by the "storage-storage-home", "storage-storage-home-data", "storage-storage-oc" and "storage-storage-oc-data" containers. Using the `owncloud/ocis` docker image, the volume would need to be hooked in the `/var/lib/ocis` folder inside the containers. - -- tree provided by a POSIX filesystem - - file layout is mapped to the old ownCloud 10 layout - - the root of tree for a user on disk is prefixed with `/path/to/data//files/` - - efficient path by id lookup - - all files and folders get assigned a uuid in the extended attributes - - when starting the storage provider it will walk all files to populate a redis kv store for uuid to path lookup - - slow to boot trees with lots of nodes - - build in ETag propagation - - ETags are calculated based on mtime - - mtime is propagated by the storage driver - - changes bypassing ocis are not picked up until a restart of the storage provider - - no subtree accounting, same options as for local storage - - efficient rename - - TODO [update the kv store for path lookup](https://github.com/cs3org/reva/issues/985), this is an O(n) operation - - arbitrary metadata using extended attributes -- grant persistence - - using custom ACLs that are stored as extended attributes - - a grant corresponds to one extended attribute of 40-100 bytes, effectively limiting the number of shares to ~100-40 - - extended attributes have varying limitations, based on the underlying filesystem - - the linux kernel imposes a limit of 255bytes per name and 64KiB per value - - ext2/3/4: total bytes for all attributes of a file is limited to 4KiB (a filesystem block) - - xfs: limit of 64KiB per value - - btrfs: total bytes used for the name, value, and implementation overhead bytes 16KiB (the default filesystem nodesize value) - - does not require OS level integration -- built in trash - - trashed files are moved to `/path/to/data//files_trashbin/` - - trashed files are appended a timestamp `.d`, which [breaks trashing of files that reach the filesystems specific name limit](https://github.com/owncloud/core/issues/28095) -- built in versions - - file versions are stored in `/path/to/data//files_versions/` - - file versions are appended a timestamp `.d`, which [breaks versioning of files that reach the filesystems specific name limit](https://github.com/owncloud/core/issues/28095) - ### EOS Storage Driver The CERN eos storage has evolved with ownCloud and natively supports id based lookup, ETag propagation, subtree size accounting, sharing, trash and versions. To use it you need to change the default configuration of the `storage storage-home` command (or have a look at the Makefile ̀ eos-start` target): diff --git a/docs/ocis/deployment/oc10_ocis_parallel.md b/docs/ocis/deployment/oc10_ocis_parallel.md index a8ccac2df54..f0d210d8981 100644 --- a/docs/ocis/deployment/oc10_ocis_parallel.md +++ b/docs/ocis/deployment/oc10_ocis_parallel.md @@ -20,7 +20,7 @@ geekdocFilePath: oc10_ocis_parallel.md - ownCloud 10 is configured to synchronize users from the LDAP server - ownCloud 10 is used to use OpenID Connect for authentication with Keycloak - oCIS running behind Traefik as reverse proxy - - oCIS is using the ownCloud storage driver on the same files and same database as ownCloud 10 + - oCIS is using the ownCloudSQL storage driver on the same files and same database as ownCloud 10 - oCIS is using Keycloak as OpenID Connect provider - oCIS is using the LDAP server as user backend - All requests to both oCIS and oC10 are routed through the oCIS proxy and will be routed based on an OIDC claim to one of them. Therefore admins can change on a user basis in the LDAP which backend is used. diff --git a/storage/pkg/config/config.go b/storage/pkg/config/config.go index 8a93c6dcbb1..f2c7c2f01cb 100644 --- a/storage/pkg/config/config.go +++ b/storage/pkg/config/config.go @@ -218,7 +218,6 @@ type PublicStorage struct { type StorageConfig struct { EOS DriverEOS `ocisConfig:"eos"` Local DriverCommon `ocisConfig:"local"` - OwnCloud DriverOwnCloud `ocisConfig:"owncloud"` OwnCloudSQL DriverOwnCloudSQL `ocisConfig:"owncloud_sql"` S3 DriverS3 `ocisConfig:"s3"` S3NG DriverS3NG `ocisConfig:"s3ng"` @@ -306,15 +305,6 @@ type DriverOCIS struct { ServiceUserUUID string `ocisConfig:"service_user_uuid"` } -// DriverOwnCloud defines the available ownCloud storage driver configuration. -type DriverOwnCloud struct { - DriverCommon - - UploadInfoDir string `ocisConfig:"upload_info_dir"` - Redis string `ocisConfig:"redis"` - Scan bool `ocisConfig:"scan"` -} - // DriverOwnCloudSQL defines the available ownCloudSQL storage driver configuration. type DriverOwnCloudSQL struct { DriverCommon @@ -584,10 +574,6 @@ func structMappings(cfg *Config) []shared.EnvBinding { EnvVars: []string{"STORAGE_USERS_DRIVER"}, Destination: &cfg.Reva.StorageUsers.Driver, }, - { - EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR"}, - Destination: &cfg.Reva.UserStorage.OwnCloud.Root, - }, { EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_ROOT"}, Destination: &cfg.Reva.UserStorage.OCIS.Root, @@ -1535,29 +1521,6 @@ func structMappings(cfg *Config) []shared.EnvBinding { EnvVars: []string{"STORAGE_USERS_DRIVER_OCIS_SERVICE_USER_UUID"}, Destination: &cfg.Reva.UserStorage.OCIS.ServiceUserUUID, }, - - // driver owncloud - { - EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_UPLOADINFO_DIR"}, - Destination: &cfg.Reva.UserStorage.OwnCloud.UploadInfoDir, - }, - { - EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_SHARE_FOLDER"}, - Destination: &cfg.Reva.UserStorage.OwnCloud.ShareFolder, - }, - { - EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_SCAN"}, - Destination: &cfg.Reva.UserStorage.OwnCloud.Scan, - }, - { - EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_REDIS_ADDR"}, - Destination: &cfg.Reva.UserStorage.OwnCloud.Redis, - }, - { - EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUD_LAYOUT"}, - Destination: &cfg.Reva.UserStorage.OwnCloud.UserLayout, - }, - // driver owncloud sql { EnvVars: []string{"STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR"}, diff --git a/storage/pkg/config/defaultconfig.go b/storage/pkg/config/defaultconfig.go index 16227e43e45..24925be5fa9 100644 --- a/storage/pkg/config/defaultconfig.go +++ b/storage/pkg/config/defaultconfig.go @@ -112,17 +112,6 @@ func DefaultConfig() *Config { UserLayout: "{{.Username}}", EnableHome: false, }, - OwnCloud: DriverOwnCloud{ - DriverCommon: DriverCommon{ - Root: path.Join(defaults.BaseDataPath(), "storage", "owncloud"), - ShareFolder: defaultShareFolder, - UserLayout: defaultUserLayout, - EnableHome: false, - }, - UploadInfoDir: path.Join(defaults.BaseDataPath(), "storage", "uploadinfo"), - Redis: ":6379", - Scan: true, - }, OwnCloudSQL: DriverOwnCloudSQL{ DriverCommon: DriverCommon{ Root: path.Join(defaults.BaseDataPath(), "storage", "owncloud"), @@ -196,7 +185,6 @@ func DefaultConfig() *Config { Local: DriverCommon{ Root: path.Join(defaults.BaseDataPath(), "storage", "local", "metadata"), }, - OwnCloud: DriverOwnCloud{}, OwnCloudSQL: DriverOwnCloudSQL{}, S3: DriverS3{ DriverCommon: DriverCommon{}, diff --git a/tests/acceptance/docker/Makefile b/tests/acceptance/docker/Makefile index 6e3e7962321..e6b9ae25d17 100644 --- a/tests/acceptance/docker/Makefile +++ b/tests/acceptance/docker/Makefile @@ -66,9 +66,6 @@ help: @echo -e "${GREEN}Run full ownCloud test suites against oCIS with s3ng storage:${RESET}\n" @echo -e "\tmake Core-API-Tests-s3ng-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}" @echo - @echo -e "${GREEN}Run full ownCloud test suites against oCIS with ownCloud storage:${RESET}\n" - @echo -e "\tmake Core-API-Tests-owncloud-storage-${RED}X${RESET}\t\t${BLUE}run test suite number X, where ${RED}X = 1 .. 10${RESET}" - @echo @echo -e "${GREEN}Run an oCIS feature test against oCIS with oCIS storage:${RESET}\n" @echo -e "\tmake test-ocis-feature-ocis-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" @echo @@ -81,9 +78,6 @@ help: @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}" @echo - @echo -e "${GREEN}Run an oCIS feature test against oCIS with owncloud storage:${RESET}\n" - @echo -e "\tmake test-ocis-feature-owncloud-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" - @echo @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\texample: ${RED}tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature${RESET}" @echo @@ -99,9 +93,6 @@ help: @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\texample: ${RED}tests/acceptance/features/apiAuth/cors.feature${RESET}" @echo - @echo -e "${GREEN}Run an ownCloud feature test against oCIS with owncloud storage:${RESET}\n" - @echo -e "\tmake test-oc10-feature-owncloud-storage ${YELLOW}BEHAT_FEATURE='...'${RESET}\t${BLUE}run single feature test${RESET}" - @echo @echo -e "\twhere ${YELLOW}BEHAT_FEATURE='...'${RESET} contains a relative path to the feature definition." @echo -e "\texample: ${RED}tests/acceptance/features/apiAuth/cors.feature${RESET}" @echo @@ -130,13 +121,6 @@ test-ocis-feature-s3ng-storage: ## test a ocis feature with s3ng storage, usage: START_CEPH=1 \ $(MAKE) --no-print-directory testSuite -.PHONY: test-ocis-feature-owncloud-storage -test-ocis-feature-owncloud-storage: ## test a ocis feature with oc10 storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature:10' - @TEST_SOURCE=ocis \ - STORAGE_DRIVER=owncloud \ - BEHAT_FEATURE=$(BEHAT_FEATURE) \ - $(MAKE) --no-print-directory testSuite - .PHONY: test-oc10-feature-ocis-storage test-oc10-feature-ocis-storage: ## test a oC10 feature with oCIS storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature' @TEST_SOURCE=oc10 \ @@ -152,13 +136,6 @@ test-oc10-feature-s3ng-storage: ## test a oC10 feature with s3ng storage, usage: START_CEPH=1 \ $(MAKE) --no-print-directory testSuite -.PHONY: test-oc10-feature-owncloud-storage -test-oc10-feature-owncloud-storage: ## test a oC10 feature with oc10 storage, usage: make ... BEHAT_FEATURE='tests/acceptance/features/apiAuth/cors.feature' - @TEST_SOURCE=oc10 \ - STORAGE_DRIVER=owncloud \ - BEHAT_FEATURE=$(BEHAT_FEATURE) \ - $(MAKE) --no-print-directory testSuite - .PHONY: localApiTests-apiAccountsHashDifficulty-owncloud localApiTests-apiAccountsHashDifficulty-owncloud: ## run apiAccountsHashDifficulty test suite with owncloud storage @TEST_SOURCE=ocis \ @@ -173,15 +150,6 @@ localApiTests-apiAccountsHashDifficulty-ocis: ## run apiAccountsHashDifficulty t BEHAT_SUITE=apiAccountsHashDifficulty \ $(MAKE) --no-print-directory testSuite -targets = $(addprefix Core-API-Tests-owncloud-storage-,$(PARTS)) -.PHONY: $(targets) -$(targets): - @$(eval RUN_PART=$(shell echo "$@" | tr -dc '0-9')) - @TEST_SOURCE=oc10 \ - STORAGE_DRIVER=owncloud \ - RUN_PART=$(RUN_PART) \ - $(MAKE) --no-print-directory testSuite - targets = $(addprefix Core-API-Tests-ocis-storage-,$(PARTS)) .PHONY: $(targets) $(targets): diff --git a/tests/acceptance/docker/src/ocis-base.yml b/tests/acceptance/docker/src/ocis-base.yml index 94e26e0cd4c..f27450fcf08 100644 --- a/tests/acceptance/docker/src/ocis-base.yml +++ b/tests/acceptance/docker/src/ocis-base.yml @@ -6,7 +6,6 @@ services: OCIS_URL: "https://ocis-server:9200" STORAGE_USERS_DRIVER: $STORAGE_DRIVER STORAGE_USERS_DRIVER_LOCAL_ROOT: /srv/app/tmp/ocis/local/root - STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR: /srv/app/tmp/ocis/owncloud/data STORAGE_USERS_DRIVER_OCIS_ROOT: /srv/app/tmp/ocis/storage/users STORAGE_METADATA_DRIVER_OCIS_ROOT: /srv/app/tmp/ocis/storage/metadata STORAGE_SHARING_USER_JSON_FILE: /srv/app/tmp/ocis/shares.json