Merge pull request #4562 from aduffeck/decomposed-based-posixfs

Decomposed based posix filesystem driver

.drone.star

@@ -1,5 +1,5 @@
 # images
-OC_CI_GOLANG = "owncloudci/golang:1.21"
+OC_CI_GOLANG = "owncloudci/golang:1.22@sha256:bc1ff4ac994a432146b0424207ca89985491496fcc534156cad5ad5d9e7e216d"
 OC_CI_ALPINE = "owncloudci/alpine:latest"
 OSIXIA_OPEN_LDAP = "osixia/openldap:1.3.0"
 REDIS = "redis:6-alpine"
@@ -103,7 +103,7 @@ def cephService():
 def main(ctx):
     # In order to run specific parts only, specify the parts as
     # ocisIntegrationTests(6, [1, 4])     - this will only run 1st and 4th parts
-    # implemented for: ocisIntegrationTests and s3ngIntegrationTests
+    # implemented for: ocisIntegrationTests, posixfsIntegrationTests and s3ngIntegrationTests
     return [
         checkStarlark(),
         checkGoGenerate(),
@@ -117,7 +117,7 @@ def main(ctx):
         cs3ApiValidatorS3NG(),
         # virtual views don't work on edge at the moment
         #virtualViews(),
-    ] + ocisIntegrationTests(6) + s3ngIntegrationTests(12)
+    ] + ocisIntegrationTests(6) + s3ngIntegrationTests(12) + posixfsIntegrationTests(6)
 
 def coverage():
     return {
@@ -705,6 +705,83 @@ def s3ngIntegrationTests(parallelRuns, skipExceptParts = []):
 
     return pipelines
 
+def posixfsIntegrationTests(parallelRuns, skipExceptParts = []):
+    pipelines = []
+    debugPartsEnabled = (len(skipExceptParts) != 0)
+    for runPart in range(1, parallelRuns + 1):
+        if debugPartsEnabled and runPart not in skipExceptParts:
+            continue
+
+        pipelines.append(
+            {
+                "kind": "pipeline",
+                "type": "docker",
+                "name": "posixfs-integration-tests-%s" % runPart,
+                "platform": {
+                    "os": "linux",
+                    "arch": "amd64",
+                },
+                "trigger": {
+                    "ref": [
+                        "refs/heads/master",
+                        "refs/heads/edge",
+                        "refs/pull/**",
+                    ],
+                },
+                "steps": [
+                    makeStep("build-ci"),
+                    {
+                        "name": "revad-services",
+                        "image": OC_CI_GOLANG,
+                        "detach": True,
+                        "commands": [
+                            "cd /drone/src/tests/oc-integration-tests/drone/",
+                            "/drone/src/cmd/revad/revad -c frontend.toml &",
+                            "/drone/src/cmd/revad/revad -c gateway.toml &",
+                            "/drone/src/cmd/revad/revad -c shares.toml &",
+                            "/drone/src/cmd/revad/revad -c storage-shares.toml &",
+                            "/drone/src/cmd/revad/revad -c machine-auth.toml &",
+                            "/drone/src/cmd/revad/revad -c storage-users-posixfs.toml &",
+                            "/drone/src/cmd/revad/revad -c storage-publiclink.toml &",
+                            "/drone/src/cmd/revad/revad -c permissions-ocis-ci.toml &",
+                            "/drone/src/cmd/revad/revad -c ldap-users.toml",
+                        ],
+                    },
+                    cloneApiTestReposStep(),
+                    {
+                        "name": "APIAcceptanceTestsPosixStorage",
+                        "image": OC_CI_PHP,
+                        "commands": [
+                            "cd /drone/src/tmp/testrunner",
+                            "make test-acceptance-from-core-api",
+                        ],
+                        "environment": {
+                            "TEST_SERVER_URL": "http://revad-services:20080",
+                            "OCIS_REVA_DATA_ROOT": "/drone/src/tmp/reva/data/",
+                            "DELETE_USER_DATA_CMD": "rm -rf /drone/src/tmp/reva/data/users/* /drone/src/tmp/reva/data/indexes/by-type/*",
+                            "STORAGE_DRIVER": "ocis",
+                            "SKELETON_DIR": "/drone/src/tmp/testing/data/apiSkeleton",
+                            "TEST_WITH_LDAP": "true",
+                            "REVA_LDAP_HOSTNAME": "ldap",
+                            "TEST_REVA": "true",
+                            "SEND_SCENARIO_LINE_REFERENCES": "true",
+                            "BEHAT_FILTER_TAGS": "~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@provisioning_api-app-required&&~@preview-extension-required&&~@local_storage&&~@skipOnOcis-OCIS-Storage&&~@skipOnGraph&&~@caldav&&~@carddav&&~@skipOnReva&&~@env-config",
+                            "DIVIDE_INTO_NUM_PARTS": parallelRuns,
+                            "RUN_PART": runPart,
+                            "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-on-POSIX-storage.md",
+                        },
+                    },
+                ],
+                "services": [
+                    redisService(),
+                    ldapService(),
+                ],
+                "depends_on": ["unit-test-coverage"],
+            },
+        )
+
+    return pipelines
+
 def checkStarlark():
     return {
         "kind": "pipeline",

.mockery.yaml

@@ -29,6 +29,9 @@ packages:
     github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree:
         interfaces:
             Blobstore:
+    github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/usermapper:
+        interfaces:
+            Mapper:
     github.com/cs3org/reva/v2/pkg/storage/utils/indexer:
         interfaces:
             Indexer:

changelog/unreleased/improve-posixfs.md

@@ -0,0 +1,5 @@
+Enhancement: Improve posixfs storage driver
+
+Improve the posixfs storage driver by fixing several issues and adding missing features. 
+
+https://github.com/cs3org/reva/pull/4562

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/go-micro/plugins/v4/events/natsjs v1.2.2-0.20231215124540-f7f8d3274bf9
 	github.com/go-micro/plugins/v4/server/http v1.2.2
 	github.com/go-micro/plugins/v4/store/nats-js v1.2.0
+	github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20231226212146-94a49ba3e06e
 	github.com/go-micro/plugins/v4/store/redis v1.2.1
 	github.com/go-playground/locales v0.14.1
 	github.com/go-playground/universal-translator v0.18.1
@@ -61,13 +62,15 @@ require (
 	github.com/onsi/ginkgo/v2 v2.15.0
 	github.com/onsi/gomega v1.31.1
 	github.com/owncloud/ocis/v2 v2.0.0
+	github.com/pablodz/inotifywaitgo v0.0.6
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/xattr v0.4.9
 	github.com/prometheus/alertmanager v0.26.0
 	github.com/prometheus/client_golang v1.18.0
 	github.com/rogpeppe/go-internal v1.12.0
 	github.com/rs/cors v1.10.1
 	github.com/rs/zerolog v1.32.0
+	github.com/segmentio/kafka-go v0.4.47
 	github.com/sethvargo/go-password v0.2.0
 	github.com/shamaton/msgpack/v2 v2.1.1
 	github.com/stretchr/testify v1.8.4
@@ -88,7 +91,7 @@ require (
 	golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3
 	golang.org/x/oauth2 v0.17.0
 	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.17.0
+	golang.org/x/sys v0.19.0
 	golang.org/x/term v0.17.0
 	golang.org/x/text v0.14.0
 	google.golang.org/genproto v0.0.0-20240205150955-31a09d347014
@@ -136,7 +139,7 @@ require (
 	github.com/go-micro/plugins/v4/registry/mdns v1.2.0 // indirect
 	github.com/go-micro/plugins/v4/registry/memory v1.2.0 // indirect
 	github.com/go-micro/plugins/v4/registry/nats v1.2.1 // indirect
-	github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20231226212146-94a49ba3e06e // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/errors v0.20.4 // indirect
 	github.com/go-openapi/strfmt v0.21.7 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
@@ -185,6 +188,7 @@ require (
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 	github.com/pkg/term v1.2.0-beta.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
@@ -195,14 +199,18 @@ require (
 	github.com/rs/xid v1.5.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.14 // indirect
+	github.com/tklauser/numcpus v0.8.0 // indirect
 	github.com/urfave/cli/v2 v2.25.7 // indirect
 	github.com/xanzy/ssh-agent v0.3.2 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.12 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect
 	go.mongodb.org/mongo-driver v1.11.3 // indirect