diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 8f9e5dd3c71..ab7c6d378b7 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -26,6 +26,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Changed - Improved loading speed of the annotation list. [#7410](https://github.com/scalableminds/webknossos/pull/7410) +- Improved loading speed for the users list. [#7466](https://github.com/scalableminds/webknossos/pull/7466) - Admins and Team Managers can now also download job exports for jobs of other users, if they have the link. [#7462](https://github.com/scalableminds/webknossos/pull/7462) - Updated some dependencies of the backend code (play 2.9, sbt 1.9, minor upgrades for others) for optimized performance. [#7366](https://github.com/scalableminds/webknossos/pull/7366) - Processing jobs can now be distributed to multiple webknossos-workers with finer-grained configurability. Compare migration guide. [#7463](https://github.com/scalableminds/webknossos/pull/7463) diff --git a/app/controllers/UserController.scala b/app/controllers/UserController.scala index 1c372b31765..a57974d3782 100755 --- a/app/controllers/UserController.scala +++ b/app/controllers/UserController.scala @@ -241,11 +241,14 @@ class UserController @Inject()(userService: UserService, isAdmin: Option[Boolean] ): Action[AnyContent] = sil.SecuredAction.async { implicit request => for { - users <- userDAO.findAllWithFilters(isEditable, isTeamManagerOrAdmin, isAdmin, request.identity) - js <- Fox.serialCombined(users.sortBy(_.lastName.toLowerCase))(u => userService.publicWrites(u, request.identity)) - } yield { - Ok(Json.toJson(js)) - } + (users, userCompactInfos) <- userDAO.findAllCompactWithFilters(isEditable, + isTeamManagerOrAdmin, + isAdmin, + request.identity) + zipped = users.zip(userCompactInfos) + js <- Fox.serialCombined(zipped.sortBy(_._1.lastName.toLowerCase))(u => + userService.publicWritesCompact(u._1, u._2)) + } yield Ok(Json.toJson(js)) } private val userUpdateReader = diff --git a/app/models/annotation/Annotation.scala b/app/models/annotation/Annotation.scala index cea330951d1..ae633e0dc9d 100755 --- a/app/models/annotation/Annotation.scala +++ b/app/models/annotation/Annotation.scala @@ -307,9 +307,6 @@ class AnnotationDAO @Inject()(sqlClient: SqlClient, annotationLayerDAO: Annotati } yield parsed } - private def parseObjectIdArray(objectIdArray: String): Seq[ObjectId] = - Option(objectIdArray).map(_.split(",").map(id => ObjectId(id))).getOrElse(Array[ObjectId]()).toSeq - def findAllListableExplorationals( isFinished: Option[Boolean], forUser: Option[ObjectId], @@ -334,9 +331,9 @@ class AnnotationDAO @Inject()(sqlClient: SqlClient, annotationLayerDAO: Annotati u.firstname, u.lastname, a.othersmayedit, - STRING_AGG(t._id, ',') AS team_ids, - STRING_AGG(t.name, ',') AS team_names, - STRING_AGG(t._organization, ',') AS team_orgs, + ARRAY_REMOVE(ARRAY_AGG(t._id), null) AS team_ids, + ARRAY_REMOVE(ARRAY_AGG(t.name), null) AS team_names, + ARRAY_REMOVE(ARRAY_AGG(t._organization), null) AS team_orgs, a.modified, a.tags, a.state, @@ -345,10 +342,10 @@ class AnnotationDAO @Inject()(sqlClient: SqlClient, annotationLayerDAO: Annotati a.visibility, a.tracingtime, o.name, - STRING_AGG(al.tracingid, ',') AS tracing_ids, - STRING_AGG(al.name, ',') AS tracing_names, - STRING_AGG(al.typ :: varchar, ',') AS tracing_typs, - ARRAY_AGG(al.statistics) AS annotation_layer_statistics + ARRAY_REMOVE(ARRAY_AGG(al.tracingid), null) AS tracing_ids, + ARRAY_REMOVE(ARRAY_AGG(al.name), null) AS tracing_names, + ARRAY_REMOVE(ARRAY_AGG(al.typ :: varchar), null) AS tracing_typs + ARRAY_REMOVE(ARRAY_AGG(al.statistics), null) AS annotation_layer_statistics FROM webknossos.annotations as a LEFT JOIN webknossos.users_ u ON u._id = a._user @@ -401,9 +398,9 @@ class AnnotationDAO @Inject()(sqlClient: SqlClient, annotationLayerDAO: Annotati ownerFirstName = r._5, ownerLastName = r._6, othersMayEdit = r._7, - teamIds = parseObjectIdArray(r._8), - teamNames = Option(r._9).map(_.split(",")).getOrElse(Array[String]()).toSeq, - teamOrganizationIds = parseObjectIdArray(r._10), + teamIds = parseArrayLiteral(r._8).map(ObjectId(_)), + teamNames = parseArrayLiteral(r._9), + teamOrganizationIds = parseArrayLiteral(r._10).map(ObjectId(_)), modified = r._11, tags = parseArrayLiteral(r._12).toSet, state = AnnotationState.fromString(r._13).getOrElse(AnnotationState.Active), @@ -412,9 +409,9 @@ class AnnotationDAO @Inject()(sqlClient: SqlClient, annotationLayerDAO: Annotati visibility = AnnotationVisibility.fromString(r._16).getOrElse(AnnotationVisibility.Internal), tracingTime = Option(r._17), organizationName = r._18, - tracingIds = Option(r._19).map(_.split(",")).getOrElse(Array[String]()).toSeq, - annotationLayerNames = Option(r._20).map(_.split(",")).getOrElse(Array[String]()).toSeq, - annotationLayerTypes = Option(r._21).map(_.split(",")).getOrElse(Array[String]()).toSeq, + tracingIds = parseArrayLiteral(r._19), + annotationLayerNames = parseArrayLiteral(r._20), + annotationLayerTypes = parseArrayLiteral(r._21), annotationLayerStatistics = parseArrayLiteral(r._22).map(layerStats => Json.parse(layerStats).validate[JsObject].getOrElse(Json.obj())) ) diff --git a/app/models/user/User.scala b/app/models/user/User.scala index 5bbd77550e2..fa065a98701 100644 --- a/app/models/user/User.scala +++ b/app/models/user/User.scala @@ -12,12 +12,14 @@ import com.scalableminds.webknossos.schema.Tables._ import javax.inject.Inject import models.team._ import play.api.libs.json._ +import slick.jdbc.GetResult import slick.jdbc.PostgresProfile.api._ import slick.jdbc.TransactionIsolation.Serializable import slick.lifted.Rep import utils.sql.{SQLDAO, SimpleSQLDAO, SqlClient, SqlToken} import utils.ObjectId +import java.sql.Timestamp import scala.concurrent.ExecutionContext object User { @@ -60,6 +62,57 @@ case class User( } +case class UserCompactInfo( + _id: String, + _multiUserId: String, + email: String, + firstname: String, + lastname: String, + userConfiguration: String, + isAdmin: Boolean, + isOrganizationOwner: Boolean, + isDatasetManager: Boolean, + isDeactivated: Boolean, + teamIdsAsArrayLiteral: String, + teamNamesAsArrayLiteral: String, + teamManagersAsArrayLiteral: String, + experienceValuesAsArrayLiteral: String, + experienceDomainsAsArrayLiteral: String, + lastActivity: Timestamp, + organization_id: String, + organization_name: String, + novelUserExperienceInfos: String, + selectedTheme: String, + created: Timestamp, + lastTaskTypeId: Option[String], + isSuperUser: Boolean, + isEmailVerified: Boolean, + isEditable: Boolean +) { + def toUser(implicit ec: ExecutionContext): Fox[User] = + for { + userConfiguration <- Fox.box2Fox(parseAndValidateJson[JsObject](userConfiguration)) + } yield { + User( + ObjectId(_id), + ObjectId(_multiUserId), + ObjectId(organization_id), + firstname, + lastname, + Instant.fromSql(lastActivity), + userConfiguration, + LoginInfo(User.default_login_provider_id, _id), + isAdmin, + isOrganizationOwner, + isDatasetManager, + isDeactivated, + isUnlisted = false, + Instant.fromSql(created), + lastTaskTypeId.map(ObjectId(_)) + ) + } +} + class UserDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) extends SQLDAO[User, UsersRow, Users](sqlClient) { protected val collection = Users @@ -92,14 +145,18 @@ class UserDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) } override protected def readAccessQ(requestingUserId: ObjectId) = - q"""(_id in (select _user from webknossos.user_team_roles where _team in (select _team from webknossos.user_team_roles where _user = $requestingUserId and isTeamManager))) - or (_organization in (select _organization from webknossos.users_ where _id = $requestingUserId and isAdmin)) - or _id = $requestingUserId""" + readAccessQWithPrefix(requestingUserId, SqlToken.raw("")) + + protected def readAccessQWithPrefix(requestingUserId: ObjectId, userPrefix: SqlToken) = + q"""(${userPrefix}_id in (select _user from webknossos.user_team_roles where _team in (select _team from webknossos.user_team_roles where _user = $requestingUserId and isTeamManager))) + or (${userPrefix}_organization in (select _organization from webknossos.users_ where _id = $requestingUserId and isAdmin)) + or ${userPrefix}_id = $requestingUserId""" override protected def deleteAccessQ(requestingUserId: ObjectId) = q"_organization in (select _organization from webknossos.users_ where _id = $requestingUserId and isAdmin)" - private def listAccessQ(requestingUserId: ObjectId) = - q"""(${readAccessQ(requestingUserId)}) + private def listAccessQ(requestingUserId: ObjectId) = listAccessQWithPrefix(requestingUserId, SqlToken.raw("")) + private def listAccessQWithPrefix(requestingUserId: ObjectId, prefix: SqlToken) = + q"""(${readAccessQWithPrefix(requestingUserId, prefix)}) and ( isUnlisted = false @@ -130,17 +187,18 @@ class UserDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) def buildSelectionPredicates(isEditableOpt: Option[Boolean], isTeamManagerOrAdminOpt: Option[Boolean], isAdminOpt: Option[Boolean], - requestingUser: User)(implicit ctx: DBAccessContext): Fox[SqlToken] = + requestingUser: User, + userPrefix: SqlToken)(implicit ctx: DBAccessContext): Fox[SqlToken] = for { - accessQuery <- accessQueryFromAccessQ(listAccessQ) + accessQuery <- accessQueryFromAccessQWithPrefix(listAccessQWithPrefix, userPrefix) editablePredicate = isEditableOpt match { case Some(isEditable) => val usersInTeamsManagedByRequestingUser = q"(SELECT _user FROM webknossos.user_team_roles WHERE _team IN (SELECT _team FROM webknossos.user_team_roles WHERE _user = ${requestingUser._id} AND isTeamManager)))" if (isEditable) { - q"(_id IN $usersInTeamsManagedByRequestingUser OR (${requestingUser.isAdmin} AND _organization = ${requestingUser._organization})" + q"(${userPrefix}_id IN $usersInTeamsManagedByRequestingUser OR (${requestingUser.isAdmin} AND ${userPrefix}_organization = ${requestingUser._organization})" } else { - q"(_id NOT IN $usersInTeamsManagedByRequestingUser AND (NOT (${requestingUser.isAdmin} AND _organization = ${requestingUser._organization}))" + q"(${userPrefix}_id NOT IN $usersInTeamsManagedByRequestingUser AND (NOT (${requestingUser.isAdmin} AND ${userPrefix}_organization = ${requestingUser._organization}))" } case None => q"${true}" } @@ -148,13 +206,13 @@ class UserDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) case Some(isTeamManagerOrAdmin) => val teamManagers = q"(SELECT _user FROM webknossos.user_team_roles WHERE isTeamManager)" if (isTeamManagerOrAdmin) { - q"_id IN $teamManagers OR isAdmin" + q"${userPrefix}_id IN $teamManagers OR ${userPrefix}isAdmin" } else { - q"_id NOT IN $teamManagers AND NOT isAdmin" + q"${userPrefix}_id NOT IN $teamManagers AND NOT ${userPrefix}isAdmin" } case None => q"${true}" } - adminPredicate = isAdminOpt.map(isAdmin => q"isAdmin = $isAdmin").getOrElse(q"${true}") + adminPredicate = isAdminOpt.map(isAdmin => q"${userPrefix}isAdmin = $isAdmin").getOrElse(q"${true}") } yield q""" ($editablePredicate) AND ($isTeamManagerOrAdminPredicate) AND @@ -162,16 +220,81 @@ class UserDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContext) $accessQuery """ - def findAllWithFilters(isEditable: Option[Boolean], - isTeamManagerOrAdmin: Option[Boolean], - isAdmin: Option[Boolean], - requestingUser: User)(implicit ctx: DBAccessContext): Fox[List[User]] = + // Necessary since a tuple can only have 22 elements + implicit def GetResultUserCompactInfo(implicit e0: GetResult[String], + e1: GetResult[java.sql.Timestamp], + e2: GetResult[Boolean], + e3: GetResult[Option[String]]): GetResult[UserCompactInfo] = GetResult { prs => + import prs._ + // format: off + UserCompactInfo(<<[String],<<[String],<<[String],<<[String],<<[String],<<[String],<<[Boolean],<<[Boolean], + <<[Boolean],<<[Boolean],<<[String],<<[String],<<[String],<<[String], <<[String],<<[java.sql.Timestamp],<<[String], + <<[String],<<[String],<<[String],<<[java.sql.Timestamp],< + Json.obj( + "id" -> parseArrayLiteral(userCompactInfo.teamIdsAsArrayLiteral)(idx), + "name" -> parseArrayLiteral(userCompactInfo.teamNamesAsArrayLiteral)(idx), + "isTeamManager" -> parseArrayLiteral(userCompactInfo.teamManagersAsArrayLiteral)(idx).toBoolean + )) + experienceJson = Json.obj( + parseArrayLiteral(userCompactInfo.experienceValuesAsArrayLiteral).zipWithIndex + .filter(valueAndIndex => tryo(valueAndIndex._1.toInt).isDefined) + .map(valueAndIndex => + (parseArrayLiteral(userCompactInfo.experienceDomainsAsArrayLiteral)(valueAndIndex._2), + Json.toJsFieldJsValueWrapper(valueAndIndex._1.toInt))): _*) + novelUserExperienceInfos <- Json.parse(userCompactInfo.novelUserExperienceInfos).validate[JsObject] + } yield { + Json.obj( + "id" -> user._id.toString, + "email" -> userCompactInfo.email, + "firstName" -> user.firstName, + "lastName" -> user.lastName, + "isAdmin" -> user.isAdmin, + "isOrganizationOwner" -> user.isOrganizationOwner, + "isDatasetManager" -> user.isDatasetManager, + "isActive" -> !user.isDeactivated, + "teams" -> teamsJson, + "experiences" -> experienceJson, + "lastActivity" -> user.lastActivity, + "isAnonymous" -> false, + "isEditable" -> userCompactInfo.isEditable, + "organization" -> userCompactInfo.organization_name, + "novelUserExperienceInfos" -> novelUserExperienceInfos, + "selectedTheme" -> userCompactInfo.selectedTheme, + "created" -> user.created, + "lastTaskTypeId" -> user.lastTaskTypeId.map(_.toString), + "isSuperUser" -> userCompactInfo.isSuperUser, + "isEmailVerified" -> userCompactInfo.isEmailVerified, + ) + } + def compactWrites(user: User): Fox[JsObject] = { implicit val ctx: DBAccessContext = GlobalAccessContext for {