Merge pull request #3331 from rhafer/idm-default-cfg

[full-ci] Switch default config from account/glauth to builtin libregraph/idm

.drone.star

@@ -81,11 +81,11 @@ config = {
         "earlyFail": True,
     },
     "accountsUITests": {
-        "skip": False,
+        "skip": True,
         "earlyFail": True,
     },
     "settingsUITests": {
-        "skip": False,
+        "skip": True,
         "earlyFail": True,
     },
     "parallelApiTests": {
@@ -111,7 +111,7 @@ config = {
         },
     },
     "graphApiTests": {
-        "skip": False,
+        "skip": True,
         "earlyFali": False,
         "numberOfParts": 10,
         "skipExceptParts": [],
@@ -505,6 +505,9 @@ def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4):
                 "name": "localApiTests-%s-%s" % (suite, storage),
                 "image": OC_CI_PHP % DEFAULT_PHP_VERSION,
                 "environment": {
+                    "TEST_WITH_GRAPH_API": "true",
+                    "PATH_TO_OCIS": "/drone/src",
+                    "PATH_TO_CORE": "/srv/app/testrunner",
                     "TEST_SERVER_URL": "https://ocis-server:9200",
                     "OCIS_REVA_DATA_ROOT": "%s" % ("/srv/app/tmp/ocis/owncloud/data/" if storage == "owncloud" else ""),
                     "SKELETON_DIR": "/srv/app/tmp/testing/data/apiSkeleton",
@@ -513,8 +516,7 @@ def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4):
                     "SEND_SCENARIO_LINE_REFERENCES": "true",
                     "STORAGE_DRIVER": storage,
                     "BEHAT_SUITE": suite,
-                    "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnOcis-%s-Storage" % ("OC" if storage == "owncloud" else "OCIS"),
-                    "PATH_TO_CORE": "/srv/app/testrunner",
+                    "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnGraph&&~@skipOnOcis-%s-Storage" % ("OC" if storage == "owncloud" else "OCIS"),
                     "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-localAPI-on-%s-storage.md" % (storage.upper()),
                     "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0,
                 },
@@ -570,6 +572,8 @@ def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4):
 
 def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", accounts_hash_difficulty = 4):
     early_fail = config["apiTests"]["earlyFail"] if "earlyFail" in config["apiTests"] else False
+    filterTags = "~@skipOnGraph&&~@skipOnOcis&&~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@local_storage&&~@skipOnOcis-%s-Storage&&~@issue-ocis-3023" % ("OC" if storage == "owncloud" else "OCIS")
+    expectedFailuresFile = "/drone/src/tests/acceptance/expected-failures-graphAPI-on-%s-storage.md" % (storage.upper())
 
     return {
         "kind": "pipeline",
@@ -586,17 +590,20 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", ac
                 "name": "oC10ApiTests-%s-storage-%s" % (storage, part_number),
                 "image": OC_CI_PHP % DEFAULT_PHP_VERSION,
                 "environment": {
+                    "TEST_WITH_GRAPH_API": "true",
+                    "PATH_TO_OCIS": "/drone/src",
+                    "PATH_TO_CORE": "/srv/app/testrunner",
                     "TEST_SERVER_URL": "https://ocis-server:9200",
                     "OCIS_REVA_DATA_ROOT": "%s" % ("/srv/app/tmp/ocis/owncloud/data/" if storage == "owncloud" else ""),
                     "SKELETON_DIR": "/srv/app/tmp/testing/data/apiSkeleton",
                     "OCIS_SKELETON_STRATEGY": "%s" % ("copy" if storage == "owncloud" else "upload"),
                     "TEST_OCIS": "true",
                     "SEND_SCENARIO_LINE_REFERENCES": "true",
                     "STORAGE_DRIVER": storage,
-                    "BEHAT_FILTER_TAGS": "~@skipOnOcis&&~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@local_storage&&~@skipOnOcis-%s-Storage&&~@issue-ocis-3023" % ("OC" if storage == "owncloud" else "OCIS"),
+                    "BEHAT_FILTER_TAGS": filterTags,
                     "DIVIDE_INTO_NUM_PARTS": number_of_parts,
                     "RUN_PART": part_number,
-                    "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-API-on-%s-storage.md" % (storage.upper()),
+                    "EXPECTED_FAILURES_FILE": expectedFailuresFile,
                     "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0,
                 },
                 "commands": [
@@ -691,7 +698,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1,
             "arch": "amd64",
         },
         "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") +
-                 ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + waitForSeleniumService() + waitForMiddlewareService() + [
+                 ocisServerWithAccounts(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + waitForSeleniumService() + waitForMiddlewareService() + [
             {
                 "name": "webUITests",
                 "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
@@ -1622,40 +1629,50 @@ def notify(ctx):
         },
     }
 
-def ocisServerWithIdp():
+def ocisServerWithAccounts(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = []):
     environment = {
-        "GRAPH_IDENTITY_BACKEND": "ldap",
-        "GRAPH_LDAP_SERVER_WRITE_ENABLED": "true",
-        "LDAP_URI": "ldaps://0.0.0.0:9235",
+        "GRAPH_IDENTITY_BACKEND": "cs3",
+        "GRAPH_LDAP_SERVER_WRITE_ENABLED": "false",
+        "LDAP_URI": "ldaps://0.0.0.0:9126",
         "LDAP_INSECURE": "true",
-        "GRAPH_LDAP_BIND_DN": "uid=libregraph,ou=sysusers,o=libregraph-idm",
-        "GRAPH_LDAP_BIND_PASSWORD": "idm",
-        "LDAP_USER_BASE_DN": "ou=users,o=libregraph-idm",
+        "LDAP_BIND_DN": "cn=admin,dc=ocis,dc=test",
+        "LDAP_BIND_PASSWORD": "admin",
+        "LDAP_USER_BASE_DN": "dc=ocis,dc=test",
         "LDAP_USER_SCHEMA_ID": "ownclouduuid",
         "LDAP_USER_SCHEMA_MAIL": "mail",
-        "LDAP_USER_SCHEMA_USERNAME": "uid",
-        "LDAP_USER_OBJECTCLASS": "inetOrgPerson",
-        "LDAP_GROUP_BASE_DN": "ou=groups,o=libregraph-idm",
-        "LDAP_GROUP_SCHEMA_ID": "ownclouduuid",
+        "LDAP_USER_SCHEMA_USERNAME": "cn",
+        "LDAP_USER_OBJECTCLASS": "posixAccount",
+        "LDAP_GROUP_BASE_DN": "dc=ocis,dc=test",
+        "LDAP_GROUP_SCHEMA_ID": "cn",
         "LDAP_GROUP_SCHEMA_MAIL": "mail",
         "LDAP_GROUP_SCHEMA_GROUPNAME": "cn",
-        "LDAP_GROUP_SCHEMA_MEMBER": "member",
-        "LDAP_GROUP_OBJECTCLASS": "groupOfNames",
-        "IDP_INSECURE": "true",
-        "IDP_LDAP_BIND_DN": "uid=idp,ou=sysusers,o=libregraph-idm",
-        "IDP_LDAP_BIND_PASSWORD": "idp",
-        "IDP_LDAP_BASE_DN": "ou=users,o=libregraph-idm",
+        "LDAP_GROUP_SCHEMA_MEMBER": "cn",
+        "LDAP_GROUP_OBJECTCLASS": "posixGroup",
+        "IDP_LDAP_BIND_DN": "cn=admin,dc=ocis,dc=test",
+        "LDAP_CACERT": "/root/.ocis/ldap/ldap.crt",
+        "IDP_LDAP_BIND_PASSWORD": "admin",
         "IDP_LDAP_LOGIN_ATTRIBUTE": "uid",
-        "PROXY_ACCOUNT_BACKEND_TYPE": "cs3",
+        "PROXY_ACCOUNT_BACKEND_TYPE": "accounts",
+        "OCS_ACCOUNT_BACKEND_TYPE": "accounts",
+        "OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,idp,nats,accounts,glauth,ocdav",
+        "OCIS_INSECURE": "true",
         "PROXY_ENABLE_BASIC_AUTH": "true",
-        "LDAP_BIND_DN": "uid=reva,ou=sysusers,o=libregraph-idm",
-        "LDAP_BIND_PASSWORD": "reva",
-        "OCS_ACCOUNT_BACKEND_TYPE": "cs3",
-        "OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,idp,nats,idm,ocdav",
+        "IDP_INSECURE": "true",
         "OCIS_LOG_LEVEL": "error",
-        "OCIS_INSECURE": "true",
         "OCIS_URL": "https://ocis-server:9200",
+        "ACCOUNTS_DEMO_USERS_AND_GROUPS": True,
+        "STORAGE_HOME_DRIVER": "%s" % (storage),
+        "STORAGE_USERS_DRIVER": "%s" % (storage),
+        "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
     }
+
+    # Pass in "default" accounts_hash_difficulty to not set this environment variable.
+    # That will allow OCIS to use whatever its built-in default is.
+    # Otherwise pass in a value from 4 to about 11 or 12 (default 4, for making regular tests fast)
+    # The high values cause lots of CPU to be used when hashing passwords, and really slow down the tests.
+    if (accounts_hash_difficulty != "default"):
+        environment["ACCOUNTS_HASH_DIFFICULTY"] = accounts_hash_difficulty
+
     return [
         {
             "name": "ocis-server",
@@ -1665,16 +1682,16 @@ def ocisServerWithIdp():
             "commands": [
                 "ocis/bin/ocis server",
             ],
-            "volumes": [stepVolumeOC10Tests],
-            "depends_on": [],
+            "volumes": volumes,
+            "depends_on": depends_on,
         },
         {
             "name": "wait-for-ocis-server",
-            "image": OC_CI_WAIT_FOR,
+            "image": OC_CI_ALPINE,
             "commands": [
-                "wait-for -it ocis-server:9200 -t 300",
+                "curl -k -u admin:admin --fail --retry-connrefused --retry 10 --retry-all-errors 'https://ocis-server:9200/graph/v1.0/users/ddc2004c-0977-11eb-9d3f-a793888cd0f8'",
             ],
-            "depends_on": [],
+            "depends_on": depends_on,
         },
     ]
 
@@ -1696,7 +1713,6 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "OCIS_LOG_LEVEL": "error",
             "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings",
             "OCIS_INSECURE": "true",
-            "ACCOUNTS_DEMO_USERS_AND_GROUPS": True,  # deprecated, remove after switching to LibreIDM
             "IDM_CREATE_DEMO_USERS": True,
         }
         wait_for_ocis = {
@@ -1710,6 +1726,8 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
     else:
         user = "33:33"
         environment = {
+            "GRAPH_IDENTITY_BACKEND": "cs3",
+            "GRAPH_LDAP_SERVER_WRITE_ENABLED": "false",
             # Keycloak IDP specific configuration
             "PROXY_OIDC_ISSUER": "https://keycloak/auth/realms/owncloud",
             "LDAP_IDP": "https://keycloak/auth/realms/owncloud",
@@ -1781,6 +1799,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on =
             "OCIS_MACHINE_AUTH_API_KEY": "change-me-please",
             "OCIS_INSECURE": "true",
             "PROXY_ENABLE_BASIC_AUTH": "true",
+            "IDM_CREATE_DEMO_USERS": True,
         }
         wait_for_ocis = {
             "name": "wait-for-ocis-server",
@@ -2508,7 +2527,7 @@ def graphApiTests(ctx, part_number = 1, number_of_parts = 1):
         },
         "steps": skipIfUnchanged(ctx, "acceptance-tests") +
                  restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") +
-                 ocisServerWithIdp() +
+                 ocisServer() +
                  cloneCoreRepos() + [
             {
                 "name": "Graph-oC10ApiTests-%s-storage-%s" % (storage, part_number),

extensions/auth-basic/pkg/command/command.go

@@ -193,6 +193,8 @@ func ldapConfigFromString(cfg config.LDAPProvider) map[string]interface{} {
 		"group_base_dn":     cfg.GroupBaseDN,
 		"user_filter":       cfg.UserFilter,
 		"group_filter":      cfg.GroupFilter,
+		"user_scope":        cfg.UserScope,
+		"group_scope":       cfg.GroupScope,
 		"user_objectclass":  cfg.UserObjectClass,
 		"group_objectclass": cfg.GroupObjectClass,
 		"login_attributes":  cfg.LoginAttributes,

extensions/auth-basic/pkg/config/config.go

@@ -66,6 +66,8 @@ type LDAPProvider struct {
 	BindPassword     string   `env:"LDAP_BIND_PASSWORD;AUTH_BASIC_LDAP_BIND_PASSWORD"`
 	UserBaseDN       string   `env:"LDAP_USER_BASE_DN;AUTH_BASIC_LDAP_USER_BASE_DN"`
 	GroupBaseDN      string   `env:"LDAP_GROUP_BASE_DN;AUTH_BASIC_LDAP_GROUP_BASE_DN"`
+	UserScope        string   `env:"LDAP_USER_SCOPE;AUTH_BASIC_LDAP_USER_SCOPE"`
+	GroupScope       string   `env:"LDAP_GROUP_SCOPE;AUTH_BASIC_LDAP_GROUP_SCOPE"`
 	UserFilter       string   `env:"LDAP_USERFILTER;AUTH_BASIC_LDAP_USERFILTER"`
 	GroupFilter      string   `env:"LDAP_GROUPFILTER;AUTH_BASIC_LDAP_USERFILTER"`
 	UserObjectClass  string   `env:"LDAP_USER_OBJECTCLASS;AUTH_BASIC_LDAP_USER_OBJECTCLASS"`

extensions/auth-basic/pkg/config/defaults/defaultconfig.go

@@ -35,31 +35,33 @@ func DefaultConfig() *config.Config {
 		AuthProvider:    "ldap",
 		AuthProviders: config.AuthProviders{
 			LDAP: config.LDAPProvider{
-				URI:              "ldaps://localhost:9126",
-				CACert:           filepath.Join(defaults.BaseDataPath(), "ldap", "ldap.crt"),
+				URI:              "ldaps://localhost:9235",
+				CACert:           filepath.Join(defaults.BaseDataPath(), "idm", "ldap.crt"),
 				Insecure:         false,
-				UserBaseDN:       "dc=ocis,dc=test",
-				GroupBaseDN:      "dc=ocis,dc=test",
-				LoginAttributes:  []string{"cn", "mail"},
+				UserBaseDN:       "ou=users,o=libregraph-idm",
+				GroupBaseDN:      "ou=groups,o=libregraph-idm",
+				UserScope:        "sub",
+				GroupScope:       "sub",
+				LoginAttributes:  []string{"uid", "mail"},
 				UserFilter:       "",
 				GroupFilter:      "",
-				UserObjectClass:  "posixAccount",
-				GroupObjectClass: "posixGroup",
-				BindDN:           "cn=reva,ou=sysusers,dc=ocis,dc=test",
+				UserObjectClass:  "inetOrgPerson",
+				GroupObjectClass: "groupOfNames",
+				BindDN:           "uid=reva,ou=sysusers,o=libregraph-idm",
 				BindPassword:     "reva",
 				IDP:              "https://localhost:9200",
 				UserSchema: config.LDAPUserSchema{
 					ID:          "ownclouduuid",
 					Mail:        "mail",
 					DisplayName: "displayname",
-					Username:    "cn",
+					Username:    "uid",
 				},
 				GroupSchema: config.LDAPGroupSchema{
-					ID:          "cn",
+					ID:          "ownclouduuid",
 					Mail:        "mail",
 					DisplayName: "cn",
 					Groupname:   "cn",
-					Member:      "cn",
+					Member:      "member",
 				},
 			},
 			JSON: config.JSONProvider{},

extensions/graph/pkg/config/defaults/defaultconfig.go

@@ -33,15 +33,15 @@ func DefaultConfig() *config.Config {
 			Insecure:     false,
 		},
 		Identity: config.Identity{
-			Backend: "cs3",
+			Backend: "ldap",
 			LDAP: config.LDAP{
-				URI:                      "ldap://localhost:9125",
-				Insecure:                 false,
-				BindDN:                   "",
-				BindPassword:             "",
+				URI:                      "ldaps://localhost:9235",
+				Insecure:                 true,
+				BindDN:                   "uid=libregraph,ou=sysusers,o=libregraph-idm",
+				BindPassword:             "idm",
 				UseServerUUID:            false,
-				WriteEnabled:             false,
-				UserBaseDN:               "ou=users,dc=ocis,dc=test",
+				WriteEnabled:             true,
+				UserBaseDN:               "ou=users,o=libregraph-idm",
 				UserSearchScope:          "sub",
 				UserFilter:               "",
 				UserObjectClass:          "inetOrgPerson",
@@ -51,7 +51,7 @@ func DefaultConfig() *config.Config {
 				// FIXME: switch this to some more widely available attribute by default
 				//        ideally this needs to	be constant for the lifetime of a users
 				UserIDAttribute:    "owncloudUUID",
-				GroupBaseDN:        "ou=groups,dc=ocis,dc=test",
+				GroupBaseDN:        "ou=groups,o=libregraph-idm",
 				GroupSearchScope:   "sub",
 				GroupFilter:        "",
 				GroupObjectClass:   "groupOfNames",

extensions/group/pkg/command/command.go

@@ -190,6 +190,8 @@ func ldapConfigFromString(cfg config.LDAPDriver) map[string]interface{} {
 		"bind_password":     cfg.BindPassword,
 		"user_base_dn":      cfg.UserBaseDN,
 		"group_base_dn":     cfg.GroupBaseDN,
+		"user_scope":        cfg.UserScope,
+		"group_scope":       cfg.GroupScope,
 		"user_filter":       cfg.UserFilter,
 		"group_filter":      cfg.GroupFilter,
 		"user_objectclass":  cfg.UserObjectClass,

extensions/group/pkg/config/config.go

@@ -67,6 +67,8 @@ type LDAPDriver struct {
 	BindPassword     string   `env:"LDAP_BIND_PASSWORD;GROUPS_LDAP_BIND_PASSWORD"`
 	UserBaseDN       string   `env:"LDAP_USER_BASE_DN;GROUPS_LDAP_USER_BASE_DN"`
 	GroupBaseDN      string   `env:"LDAP_GROUP_BASE_DN;GROUPS_LDAP_GROUP_BASE_DN"`
+	UserScope        string   `env:"LDAP_USER_SCOPE;GROUPS_LDAP_USER_SCOPE"`
+	GroupScope       string   `env:"LDAP_GROUP_SCOPE;GROUPS_LDAP_GROUP_SCOPE"`
 	UserFilter       string   `env:"LDAP_USERFILTER;GROUPS_LDAP_USERFILTER"`
 	GroupFilter      string   `env:"LDAP_GROUPFILTER;GROUPS_LDAP_USERFILTER"`
 	UserObjectClass  string   `env:"LDAP_USER_OBJECTCLASS;GROUPS_LDAP_USER_OBJECTCLASS"`

extensions/group/pkg/config/defaults/defaultconfig.go

@@ -36,31 +36,33 @@ func DefaultConfig() *config.Config {
 		Driver:                      "ldap",
 		Drivers: config.Drivers{
 			LDAP: config.LDAPDriver{
-				URI:              "ldaps://localhost:9126",
-				CACert:           filepath.Join(defaults.BaseDataPath(), "ldap", "ldap.crt"),
+				URI:              "ldaps://localhost:9235",
+				CACert:           filepath.Join(defaults.BaseDataPath(), "idm", "ldap.crt"),
 				Insecure:         false,
-				UserBaseDN:       "dc=ocis,dc=test",
-				GroupBaseDN:      "dc=ocis,dc=test",
-				LoginAttributes:  []string{"cn", "mail"},
+				UserBaseDN:       "ou=users,o=libregraph-idm",
+				GroupBaseDN:      "ou=groups,o=libregraph-idm",
+				UserScope:        "sub",
+				GroupScope:       "sub",
+				LoginAttributes:  []string{"uid", "mail"},
 				UserFilter:       "",
 				GroupFilter:      "",
-				UserObjectClass:  "posixAccount",
-				GroupObjectClass: "posixGroup",
-				BindDN:           "cn=reva,ou=sysusers,dc=ocis,dc=test",
+				UserObjectClass:  "inetOrgPerson",
+				GroupObjectClass: "groupOfNames",
+				BindDN:           "uid=reva,ou=sysusers,o=libregraph-idm",
 				BindPassword:     "reva",
 				IDP:              "https://localhost:9200",
 				UserSchema: config.LDAPUserSchema{
 					ID:          "ownclouduuid",
 					Mail:        "mail",
 					DisplayName: "displayname",
-					Username:    "cn",
+					Username:    "uid",
 				},
 				GroupSchema: config.LDAPGroupSchema{
-					ID:          "cn",
+					ID:          "ownclouduuid",
 					Mail:        "mail",
 					DisplayName: "cn",
 					Groupname:   "cn",
-					Member:      "cn",
+					Member:      "member",
 				},
 			},
 			JSON: config.JSONDriver{},

extensions/idp/pkg/config/config.go

@@ -27,7 +27,8 @@ type Config struct {
 
 // Ldap defines the available LDAP configuration.
 type Ldap struct {
-	URI string `yaml:"uri" env:"LDAP_URI;IDP_LDAP_URI"`
+	URI       string `yaml:"uri" env:"LDAP_URI;IDP_LDAP_URI"`
+	TLSCACert string `yaml:"cacert" env:"LDAP_CACERT;IDP_LDAP_TLS_CACERT"`
 
 	BindDN       string `yaml:"bind_dn" env:"LDAP_BIND_DN;IDP_LDAP_BIND_DN"`
 	BindPassword string `yaml:"bind_password" env:"LDAP_BIND_PASSWORD;IDP_LDAP_BIND_PASSWORD"`