diff --git a/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php b/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php index 32fb569a3d462..b1a795b775cc9 100644 --- a/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php +++ b/apps/files/lib/BackgroundJob/DeleteOrphanedItems.php @@ -52,34 +52,86 @@ public function run($argument) { * @param string $typeCol * @return int Number of deleted entries */ - protected function cleanUp($table, $idCol, $typeCol) { + protected function cleanUp(string $table, string $idCol, string $typeCol): int { $deletedEntries = 0; - $query = $this->connection->getQueryBuilder(); - $query->select('t1.' . $idCol) - ->from($table, 't1') - ->where($query->expr()->eq($typeCol, $query->expr()->literal('files'))) - ->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid')) - ->andWhere($query->expr()->isNull('t2.fileid')) - ->groupBy('t1.' . $idCol) - ->setMaxResults(self::CHUNK_SIZE); - $deleteQuery = $this->connection->getQueryBuilder(); $deleteQuery->delete($table) - ->where($deleteQuery->expr()->in($idCol, $deleteQuery->createParameter('objectid'))); + ->where($deleteQuery->expr()->eq($idCol, $deleteQuery->createParameter('objectid'))); + + if ($this->connection->getShardDefinition('filecache')) { + $sourceIdChunks = $this->getItemIds($table, $idCol, $typeCol, 1000); + foreach ($sourceIdChunks as $sourceIdChunk) { + $deletedSources = $this->findMissingSources($sourceIdChunk); + $deleteQuery->setParameter('objectid', $deletedSources, IQueryBuilder::PARAM_INT_ARRAY); + $deletedEntries += $deleteQuery->executeStatement(); + } + } else { + $query = $this->connection->getQueryBuilder(); + $query->select('t1.' . $idCol) + ->from($table, 't1') + ->where($query->expr()->eq($typeCol, $query->expr()->literal('files'))) + ->leftJoin('t1', 'filecache', 't2', $query->expr()->eq($query->expr()->castColumn('t1.' . $idCol, IQueryBuilder::PARAM_INT), 't2.fileid')) + ->andWhere($query->expr()->isNull('t2.fileid')) + ->groupBy('t1.' . $idCol) + ->setMaxResults(self::CHUNK_SIZE); + + $deleteQuery = $this->connection->getQueryBuilder(); + $deleteQuery->delete($table) + ->where($deleteQuery->expr()->in($idCol, $deleteQuery->createParameter('objectid'))); - $deletedInLastChunk = self::CHUNK_SIZE; - while ($deletedInLastChunk === self::CHUNK_SIZE) { - $chunk = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); - $deletedInLastChunk = count($chunk); + $deletedInLastChunk = self::CHUNK_SIZE; + while ($deletedInLastChunk === self::CHUNK_SIZE) { + $chunk = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + $deletedInLastChunk = count($chunk); - $deleteQuery->setParameter('objectid', $chunk, IQueryBuilder::PARAM_INT_ARRAY); - $deletedEntries += $deleteQuery->executeStatement(); + $deleteQuery->setParameter('objectid', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $deletedEntries += $deleteQuery->executeStatement(); + } } return $deletedEntries; } + /** + * @param string $table + * @param string $idCol + * @param string $typeCol + * @param int $chunkSize + * @return \Iterator + * @throws \OCP\DB\Exception + */ + private function getItemIds(string $table, string $idCol, string $typeCol, int $chunkSize): \Iterator { + $query = $this->connection->getQueryBuilder(); + $query->select($idCol) + ->from($table) + ->where($query->expr()->eq($typeCol, $query->expr()->literal('files'))) + ->groupBy($idCol) + ->andWhere($query->expr()->gt($idCol, $query->createParameter('min_id'))) + ->setMaxResults($chunkSize); + + $minId = 0; + while (true) { + $query->setParameter('min_id', $minId); + $rows = $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + if (count($rows) > 0) { + $minId = $rows[count($rows) - 1]; + yield $rows; + } else { + break; + } + } + } + + private function findMissingSources(array $ids): array { + $qb = $this->connection->getQueryBuilder(); + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY))); + $found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + return array_diff($ids, $found); + } + /** * Deleting orphaned system tag mappings * diff --git a/apps/files/lib/Command/DeleteOrphanedFiles.php b/apps/files/lib/Command/DeleteOrphanedFiles.php index b7101c072586f..047e26f11eca7 100644 --- a/apps/files/lib/Command/DeleteOrphanedFiles.php +++ b/apps/files/lib/Command/DeleteOrphanedFiles.php @@ -35,34 +35,29 @@ protected function configure(): void { public function execute(InputInterface $input, OutputInterface $output): int { $deletedEntries = 0; + $fileIdsByStorage = []; - $query = $this->connection->getQueryBuilder(); - $query->select('fc.fileid') - ->from('filecache', 'fc') - ->where($query->expr()->isNull('s.numeric_id')) - ->leftJoin('fc', 'storages', 's', $query->expr()->eq('fc.storage', 's.numeric_id')) - ->setMaxResults(self::CHUNK_SIZE); + $deletedStorages = array_diff($this->getReferencedStorages(), $this->getExistingStorages()); + + $deleteExtended = !$input->getOption('skip-filecache-extended'); + if ($deleteExtended) { + $fileIdsByStorage = $this->getFileIdsForStorages($deletedStorages); + } $deleteQuery = $this->connection->getQueryBuilder(); $deleteQuery->delete('filecache') - ->where($deleteQuery->expr()->eq('fileid', $deleteQuery->createParameter('objectid'))); + ->where($deleteQuery->expr()->in('storage', $deleteQuery->createParameter('storage_ids'))); - $deletedInLastChunk = self::CHUNK_SIZE; - while ($deletedInLastChunk === self::CHUNK_SIZE) { - $deletedInLastChunk = 0; - $result = $query->execute(); - while ($row = $result->fetch()) { - $deletedInLastChunk++; - $deletedEntries += $deleteQuery->setParameter('objectid', (int) $row['fileid']) - ->execute(); - } - $result->closeCursor(); + $deletedStorageChunks = array_chunk($deletedStorages, self::CHUNK_SIZE); + foreach ($deletedStorageChunks as $deletedStorageChunk) { + $deleteQuery->setParameter('storage_ids', $deletedStorageChunk, IQueryBuilder::PARAM_INT_ARRAY); + $deletedEntries += $deleteQuery->executeStatement(); } $output->writeln("$deletedEntries orphaned file cache entries deleted"); - if (!$input->getOption('skip-filecache-extended')) { - $deletedFileCacheExtended = $this->cleanupOrphanedFileCacheExtended(); + if ($deleteExtended) { + $deletedFileCacheExtended = $this->cleanupOrphanedFileCacheExtended($fileIdsByStorage); $output->writeln("$deletedFileCacheExtended orphaned file cache extended entries deleted"); } @@ -72,28 +67,63 @@ public function execute(InputInterface $input, OutputInterface $output): int { return self::SUCCESS; } - private function cleanupOrphanedFileCacheExtended(): int { - $deletedEntries = 0; - + private function getReferencedStorages(): array { $query = $this->connection->getQueryBuilder(); - $query->select('fce.fileid') - ->from('filecache_extended', 'fce') - ->leftJoin('fce', 'filecache', 'fc', $query->expr()->eq('fce.fileid', 'fc.fileid')) - ->where($query->expr()->isNull('fc.fileid')) - ->setMaxResults(self::CHUNK_SIZE); + $query->select('storage') + ->from('filecache') + ->groupBy('storage') + ->runAcrossAllShards(); + return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + } - $deleteQuery = $this->connection->getQueryBuilder(); - $deleteQuery->delete('filecache_extended') - ->where($deleteQuery->expr()->in('fileid', $deleteQuery->createParameter('idsToDelete'))); + private function getExistingStorages(): array { + $query = $this->connection->getQueryBuilder(); + $query->select('numeric_id') + ->from('storages') + ->groupBy('numeric_id'); + return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + } - $result = $query->executeQuery(); - while ($result->rowCount() > 0) { - $idsToDelete = $result->fetchAll(\PDO::FETCH_COLUMN); + /** + * @param int[] $storageIds + * @return array + */ + private function getFileIdsForStorages(array $storageIds): array { + $query = $this->connection->getQueryBuilder(); + $query->select('storage', 'fileid') + ->from('filecache') + ->where($query->expr()->in('storage', $query->createParameter('storage_ids'))); + + $result = []; + $storageIdChunks = array_chunk($storageIds, self::CHUNK_SIZE); + foreach ($storageIdChunks as $storageIdChunk) { + $query->setParameter('storage_ids', $storageIdChunk, IQueryBuilder::PARAM_INT_ARRAY); + $chunk = $query->executeQuery()->fetchAll(); + foreach ($chunk as $row) { + $result[$row['storage']][] = $row['fileid']; + } + } + return $result; + } - $deleteQuery->setParameter('idsToDelete', $idsToDelete, IQueryBuilder::PARAM_INT_ARRAY); - $deletedEntries += $deleteQuery->executeStatement(); + /** + * @param array $fileIdsByStorage + * @return int + */ + private function cleanupOrphanedFileCacheExtended(array $fileIdsByStorage): int { + $deletedEntries = 0; - $result = $query->executeQuery(); + $deleteQuery = $this->connection->getQueryBuilder(); + $deleteQuery->delete('filecache_extended') + ->where($deleteQuery->expr()->in('fileid', $deleteQuery->createParameter('file_ids'))); + + foreach ($fileIdsByStorage as $storageId => $fileIds) { + $deleteQuery->hintShardKey('storage', $storageId, true); + $fileChunks = array_chunk($fileIds, self::CHUNK_SIZE); + foreach ($fileChunks as $fileChunk) { + $deleteQuery->setParameter('file_ids', $fileChunk, IQueryBuilder::PARAM_INT_ARRAY); + $deletedEntries += $deleteQuery->executeStatement(); + } } return $deletedEntries; diff --git a/apps/files/tests/Command/DeleteOrphanedFilesTest.php b/apps/files/tests/Command/DeleteOrphanedFilesTest.php index e52f9e1e1305a..ed9a1866d2616 100644 --- a/apps/files/tests/Command/DeleteOrphanedFilesTest.php +++ b/apps/files/tests/Command/DeleteOrphanedFilesTest.php @@ -64,13 +64,19 @@ protected function tearDown(): void { } protected function getFile($fileId) { - $stmt = $this->connection->executeQuery('SELECT * FROM `*PREFIX*filecache` WHERE `fileid` = ?', [$fileId]); - return $stmt->fetchAll(); + $query = $this->connection->getQueryBuilder(); + $query->select('*') + ->from('filecache') + ->where($query->expr()->eq('fileid', $query->createNamedParameter($fileId))); + return $query->executeQuery()->fetchAll(); } protected function getMounts($storageId) { - $stmt = $this->connection->executeQuery('SELECT * FROM `*PREFIX*mounts` WHERE `storage_id` = ?', [$storageId]); - return $stmt->fetchAll(); + $query = $this->connection->getQueryBuilder(); + $query->select('*') + ->from('mounts') + ->where($query->expr()->eq('storage_id', $query->createNamedParameter($storageId))); + return $query->executeQuery()->fetchAll(); } /** diff --git a/apps/files_sharing/lib/DeleteOrphanedSharesJob.php b/apps/files_sharing/lib/DeleteOrphanedSharesJob.php index 9a052b3d126bc..bcbdbaba53165 100644 --- a/apps/files_sharing/lib/DeleteOrphanedSharesJob.php +++ b/apps/files_sharing/lib/DeleteOrphanedSharesJob.php @@ -55,6 +55,11 @@ public function __construct( * @param array $argument unused argument */ public function run($argument) { + if ($this->db->getShardDefinition('filecache')) { + $this->shardingCleanup(); + return; + } + $qbSelect = $this->db->getQueryBuilder(); $qbSelect->select('id') ->from('share', 's') @@ -96,4 +101,40 @@ public function run($argument) { }, $this->db); } while ($deleted >= self::CHUNK_SIZE && $this->time->getTime() <= $cutOff); } + + private function shardingCleanup(): void { + $qb = $this->db->getQueryBuilder(); + $qb->selectDistinct('file_source') + ->from('share', 's'); + $sourceFiles = $qb->executeQuery()->fetchAll(PDO::FETCH_COLUMN); + + $deleteQb = $this->db->getQueryBuilder(); + $deleteQb->delete('share') + ->where( + $deleteQb->expr()->in('file_source', $deleteQb->createParameter('ids'), IQueryBuilder::PARAM_INT_ARRAY) + ); + + $chunks = array_chunk($sourceFiles, self::CHUNK_SIZE); + foreach ($chunks as $chunk) { + $deletedFiles = $this->findMissingSources($chunk); + $this->atomic(function () use ($deletedFiles, $deleteQb) { + $deleteQb->setParameter('ids', $deletedFiles, IQueryBuilder::PARAM_INT_ARRAY); + $deleted = $deleteQb->executeStatement(); + $this->logger->debug("{deleted} orphaned share(s) deleted", [ + 'app' => 'DeleteOrphanedSharesJob', + 'deleted' => $deleted, + ]); + return $deleted; + }, $this->db); + } + } + + private function findMissingSources(array $ids): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY))); + $found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + return array_diff($ids, $found); + } } diff --git a/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php b/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php index 8ed88198c19fc..c40cadfbdb5f6 100644 --- a/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php +++ b/lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php @@ -289,8 +289,8 @@ public function executeStatement(?IDBConnection $connection = null): int { return $this->builder->executeStatement($connection); } - public function hintShardKey(string $column, mixed $value) { - $this->builder->hintShardKey($column, $value); + public function hintShardKey(string $column, mixed $value, bool $overwrite = false) { + $this->builder->hintShardKey($column, $value, $overwrite); return $this; } diff --git a/lib/private/DB/QueryBuilder/QueryBuilder.php b/lib/private/DB/QueryBuilder/QueryBuilder.php index c8055259f50c4..af89db7e45624 100644 --- a/lib/private/DB/QueryBuilder/QueryBuilder.php +++ b/lib/private/DB/QueryBuilder/QueryBuilder.php @@ -1366,7 +1366,7 @@ public function quoteAlias($alias) { return $this->helper->quoteColumnName($alias); } - public function hintShardKey(string $column, mixed $value) { + public function hintShardKey(string $column, mixed $value, bool $overwrite = false) { return $this; } diff --git a/lib/private/DB/QueryBuilder/Sharded/ShardedQueryBuilder.php b/lib/private/DB/QueryBuilder/Sharded/ShardedQueryBuilder.php index 6496453a1a609..650e414096e65 100644 --- a/lib/private/DB/QueryBuilder/Sharded/ShardedQueryBuilder.php +++ b/lib/private/DB/QueryBuilder/Sharded/ShardedQueryBuilder.php @@ -296,7 +296,11 @@ private function registerOrder(string $column, string $order): void { ]; } - public function hintShardKey(string $column, mixed $value) { + public function hintShardKey(string $column, mixed $value, bool $overwrite = false) { + if ($overwrite) { + $this->primaryKeys = []; + $this->shardKeys = []; + } if ($this->shardDefinition?->isKey($column)) { $this->primaryKeys[] = $value; } diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index deadcd007b1af..acf7bf22f5259 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -16,6 +16,7 @@ use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\IDBConnection; +use function Symfony\Component\Translation\t; class BackgroundCleanupJob extends TimedJob { /** @var IDBConnection */ @@ -64,6 +65,11 @@ private function getDeletedFiles(): \Iterator { } private function getOldPreviewLocations(): \Iterator { + if ($this->connection->getShardDefinition('filecache')) { + // sharding is new enough that we don't need to support this + return; + } + $qb = $this->connection->getQueryBuilder(); $qb->select('a.name') ->from('filecache', 'a') @@ -106,6 +112,15 @@ private function getNewPreviewLocations(): \Iterator { return []; } + if ($this->connection->getShardDefinition('filecache')) { + $chunks = $this->getAllPreviewIds($data['path'], 1000); + foreach ($chunks as $chunk) { + yield from $this->findMissingSources($chunk); + } + + return; + } + /* * This lovely like is the result of the way the new previews are stored * We take the md5 of the name (fileid) and split the first 7 chars. That way @@ -155,4 +170,46 @@ private function getNewPreviewLocations(): \Iterator { $cursor->closeCursor(); } + + private function getAllPreviewIds(string $previewRoot, int $chunkSize): \Iterator { + // See `getNewPreviewLocations` for some more info about the logic here + $like = $this->connection->escapeLikeParameter($previewRoot). '/_/_/_/_/_/_/_/%'; + + $qb = $this->connection->getQueryBuilder(); + $qb->select('name', 'fileid') + ->from('filecache') + ->where( + $qb->expr()->andX( + $qb->expr()->eq('storage', $qb->createNamedParameter($this->previewFolder->getStorageId())), + $qb->expr()->like('path', $qb->createNamedParameter($like)), + $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))), + $qb->expr()->gt('fileid', $qb->createParameter('min_id')), + ) + ) + ->orderBy('fileid', 'ASC') + ->setMaxResults($chunkSize); + + $minId = 0; + while (true) { + $qb->setParameter('min_id', $minId); + $rows = $qb->executeQuery()->fetchAll(); + if (count($rows) > 0) { + $minId = $rows[count($rows) - 1]['fileid']; + yield array_map(function ($row) { + return (int)$row['name']; + }, $rows); + } else { + break; + } + } + } + + private function findMissingSources(array $ids): array { + $qb = $this->connection->getQueryBuilder(); + $qb->select('fileid') + ->from('filecache') + ->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY))); + $found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); + return array_diff($ids, $found); + } } diff --git a/lib/private/Repair/CleanTags.php b/lib/private/Repair/CleanTags.php index f2fc8156f29de..b7960a1def1b6 100644 --- a/lib/private/Repair/CleanTags.php +++ b/lib/private/Repair/CleanTags.php @@ -107,7 +107,7 @@ protected function deleteOrphanFileEntries(IOutput $output) { $output, '%d tags for delete files have been removed.', 'vcategory_to_object', 'objid', - 'filecache', 'fileid', 'path_hash' + 'filecache', 'fileid', 'fileid' ); } @@ -169,16 +169,17 @@ protected function deleteOrphanEntries(IOutput $output, $repairInfo, $deleteTabl $orphanItems[] = (int) $row[$deleteId]; } + $deleteQuery = $this->connection->getQueryBuilder(); + $deleteQuery->delete($deleteTable) + ->where( + $deleteQuery->expr()->eq('type', $deleteQuery->expr()->literal('files')) + ) + ->andWhere($deleteQuery->expr()->in($deleteId, $deleteQuery->createParameter('ids'))); if (!empty($orphanItems)) { $orphanItemsBatch = array_chunk($orphanItems, 200); foreach ($orphanItemsBatch as $items) { - $qb->delete($deleteTable) - ->where( - $qb->expr()->eq('type', $qb->expr()->literal('files')) - ) - ->andWhere($qb->expr()->in($deleteId, $qb->createParameter('ids'))); - $qb->setParameter('ids', $items, IQueryBuilder::PARAM_INT_ARRAY); - $qb->execute(); + $deleteQuery->setParameter('ids', $items, IQueryBuilder::PARAM_INT_ARRAY); + $deleteQuery->executeStatement(); } } diff --git a/lib/public/DB/QueryBuilder/IQueryBuilder.php b/lib/public/DB/QueryBuilder/IQueryBuilder.php index 048de26c22a1e..b673c5ef6ec77 100644 --- a/lib/public/DB/QueryBuilder/IQueryBuilder.php +++ b/lib/public/DB/QueryBuilder/IQueryBuilder.php @@ -1036,7 +1036,7 @@ public function getColumnName($column, $tableAlias = ''); * @return $this * @since 30.0.0 */ - public function hintShardKey(string $column, mixed $value); + public function hintShardKey(string $column, mixed $value, bool $overwrite = false); /** * Set the query to run across all shards if sharding is enabled. diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index c07ec42b36b34..ccd5dba69cf05 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -146,6 +146,10 @@ public function testCleanupSystemCron() { } public function testCleanupAjax() { + if ($this->connection->getShardDefinition('filecache')) { + $this->markTestSkipped("ajax cron is not supported for sharded setups"); + return; + } $files = $this->setup11Previews(); $fileIds = array_map(function (File $f) { return $f->getId(); @@ -174,6 +178,10 @@ public function testCleanupAjax() { } public function testOldPreviews() { + if ($this->connection->getShardDefinition('filecache')) { + $this->markTestSkipped("old previews are not supported for sharded setups"); + return; + } $appdata = \OC::$server->getAppDataDir('preview'); $f1 = $appdata->newFolder('123456781'); diff --git a/tests/lib/Repair/CleanTagsTest.php b/tests/lib/Repair/CleanTagsTest.php index 5e7b82d219889..11430cc121550 100644 --- a/tests/lib/Repair/CleanTagsTest.php +++ b/tests/lib/Repair/CleanTagsTest.php @@ -142,7 +142,7 @@ protected function addTagCategory($category, $type, $user = 'TestRepairCleanTags ]) ->execute(); - return (int) $this->getLastInsertID('vcategory', 'id'); + return $qb->getLastInsertId(); } /** @@ -191,16 +191,7 @@ protected function getFileID() { ]) ->execute(); - $this->createdFile = (int) $this->getLastInsertID('filecache', 'fileid'); + $this->createdFile = $qb->getLastInsertId(); return $this->createdFile; } - - /** - * @param $tableName - * @param $idName - * @return int - */ - protected function getLastInsertID($tableName, $idName) { - return $this->connection->lastInsertId("*PREFIX*$tableName"); - } }