respect DB limits limit per statement and query

Signed-off-by: Arthur Schiwon <blizzz@arthur-schiwon.de>
nextcloud · Jan 11, 2021 · 45e3261 · 45e3261
1 parent 8b656e3
commit 45e3261
Show file tree

Hide file tree

Showing 2 changed files with 64 additions and 6 deletions.
diff --git a/apps/user_ldap/lib/Mapping/AbstractMapping.php b/apps/user_ldap/lib/Mapping/AbstractMapping.php
@@ -30,6 +30,7 @@
 use Doctrine\DBAL\Exception;
 use OC\DB\QueryBuilder\QueryBuilder;
 use OCP\DB\IPreparedStatement;
+use OCP\DB\QueryBuilder\IQueryBuilder;
 
 /**
  * Class AbstractMapping
@@ -199,19 +200,57 @@ public function getNameByDN($fdn) {
 		return $this->cache[$fdn];
 	}
 
-	public function getListOfIdsByDn(array $fdns): array {
+	protected function prepareListOfIdsQuery(array $dnList): IQueryBuilder {
 		$qb = $this->dbc->getQueryBuilder();
 		$qb->select('owncloud_name', 'ldap_dn')
 			->from($this->getTableName(false))
-			->where($qb->expr()->in('ldap_dn', $qb->createNamedParameter($fdns, QueryBuilder::PARAM_STR_ARRAY)));
-		$stmt = $qb->execute();
+			->where($qb->expr()->in('ldap_dn', $qb->createNamedParameter($dnList, QueryBuilder::PARAM_STR_ARRAY)));
+		return $qb;
+	}
 
-		$results = $stmt->fetchAll(\Doctrine\DBAL\FetchMode::ASSOCIATIVE);
-		foreach ($results as $key => $entry) {
-			unset($results[$key]);
+	protected function collectResultsFromListOfIdsQuery(IQueryBuilder $qb, array &$results): void {
+		$stmt = $qb->execute();
+		while ($entry = $stmt->fetch(\Doctrine\DBAL\FetchMode::ASSOCIATIVE)) {
 			$results[$entry['ldap_dn']] = $entry['owncloud_name'];
 			$this->cache[$entry['ldap_dn']] = $entry['owncloud_name'];
 		}
+		$stmt->closeCursor();
+	}
+
+	public function getListOfIdsByDn(array $fdns): array {
+		$totalDBParamLimit = 65000;
+		$sliceSize = 1000;
+		$maxSlices = $totalDBParamLimit / $sliceSize;
+		$results = [];
+
+		$slice = 1;
+		$fdnsSlice = count($fdns) > $sliceSize ? array_slice($fdns, 0, $sliceSize) : $fdns;
+		$qb = $this->prepareListOfIdsQuery($fdnsSlice);
+
+		while (isset($fdnsSlice[999])) {
+			// Oracle does not allow more than 1000 values in the IN list,
+			// but allows slicing
+			$slice++;
+			$fdnsSlice = array_slice($fdns, $sliceSize * ($slice - 1), $sliceSize);
+
+			if (!isset($qb)) {
+				$qb = $this->prepareListOfIdsQuery($fdnsSlice);
+				continue;
+			}
+
+			if (!empty($fdnsSlice)) {
+				$qb->orWhere($qb->expr()->in('ldap_dn', $qb->createNamedParameter($fdnsSlice, QueryBuilder::PARAM_STR_ARRAY)));
+			}
+
+			if ($slice % $maxSlices === 0) {
+				$this->collectResultsFromListOfIdsQuery($qb, $results);
+				unset($qb);
+			}
+		}
+
+		if (isset($qb)) {
+			$this->collectResultsFromListOfIdsQuery($qb, $results);
+		}
 
 		return $results;
 	}

diff --git a/apps/user_ldap/tests/Mapping/AbstractMappingTest.php b/apps/user_ldap/tests/Mapping/AbstractMappingTest.php
@@ -281,4 +281,23 @@ public function testList() {
 		$results = $mapper->getList(1, 1);
 		$this->assertSame(1, count($results));
 	}
+
+	public function testGetListOfIdsByDn() {
+		/** @var AbstractMapping $mapper */
+		list($mapper,) = $this->initTest();
+
+		$listOfDNs = [];
+		for ($i = 0; $i < 66640; $i++) {
+			// Postgres has a limit of 65535 values in a single IN list
+			$name = 'as_' . $i;
+			$dn = 'uid=' . $name . ',dc=example,dc=org';
+			$listOfDNs[] = $dn;
+			if ($i % 20 === 0) {
+				$mapper->map($dn, $name, 'fake-uuid-' . $i);
+			}
+		}
+
+		$result = $mapper->getListOfIdsByDn($listOfDNs);
+		$this->assertSame(66640 / 20, count($result));
+	}
 }