Skip to content

Commit

Permalink
Merge pull request #26264 from nextcloud/unified-search-node-19
Browse files Browse the repository at this point in the history 
[stable19] Handle limit offset and sorting in files search
  • Loading branch information
@MorrisJobke
MorrisJobke authored Apr 1, 2021
2 parents d6889a7 + 9c046ef commit 2814fc6
Show file tree
Hide file tree
Showing 9 changed files with 400 additions and 193 deletions.
10 changes: 7 additions & 3 deletions lib/private/Files/Cache/Cache.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,10 +188,10 @@ public static function cacheEntryFromData($data, IMimeTypeLoader $mimetypeLoader
}
$data['permissions'] = (int)$data['permissions'];
if (isset($data['creation_time'])) {
$data['creation_time'] = (int) $data['creation_time'];
$data['creation_time'] = (int)$data['creation_time'];
}
if (isset($data['upload_time'])) {
$data['upload_time'] = (int) $data['upload_time'];
$data['upload_time'] = (int)$data['upload_time'];
}
return new CacheEntry($data);
}
Expand Down Expand Up @@ -792,14 +792,18 @@ public function searchQuery(ISearchQuery $searchQuery) {
$query->whereStorageId();

if ($this->querySearchHelper->shouldJoinTags($searchQuery->getSearchOperation())) {
$user = $searchQuery->getUser();
if ($user === null) {
throw new \InvalidArgumentException("Searching by tag requires the user to be set in the query");
}
$query
->innerJoin('file', 'vcategory_to_object', 'tagmap', $builder->expr()->eq('file.fileid', 'tagmap.objid'))
->innerJoin('tagmap', 'vcategory', 'tag', $builder->expr()->andX(
$builder->expr()->eq('tagmap.type', 'tag.type'),
$builder->expr()->eq('tagmap.categoryid', 'tag.id')
))
->andWhere($builder->expr()->eq('tag.type', $builder->createNamedParameter('files')))
->andWhere($builder->expr()->eq('tag.uid', $builder->createNamedParameter($searchQuery->getUser()->getUID())));
->andWhere($builder->expr()->eq('tag.uid', $builder->createNamedParameter($user->getUID())));
}

$searchExpr = $this->querySearchHelper->searchOperatorToDBExpr($builder, $searchQuery->getSearchOperation());
Expand Down
171 changes: 117 additions & 54 deletions lib/private/Files/Node/Folder.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,23 @@
namespace OC\Files\Node;

use OC\DB\QueryBuilder\Literal;
use OC\Files\Search\SearchBinaryOperator;
use OC\Files\Search\SearchComparison;
use OC\Files\Search\SearchQuery;
use OC\Files\Storage\Storage;
use OCA\Files_Sharing\SharedStorage;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Cache\ICacheEntry;
use OCP\Files\Config\ICachedMountInfo;
use OCP\Files\FileInfo;
use OCP\Files\Mount\IMountPoint;
use OCP\Files\NotFoundException;
use OCP\Files\NotPermittedException;
use OCP\Files\Search\ISearchBinaryOperator;
use OCP\Files\Search\ISearchComparison;
use OCP\Files\Search\ISearchOperator;
use OCP\Files\Search\ISearchQuery;
use OCP\IUserManager;

class Folder extends Node implements \OCP\Files\Folder {
/**
Expand Down Expand Up @@ -94,8 +103,8 @@ public function isSubNode($node) {
/**
* get the content of this directory
*
* @throws \OCP\Files\NotFoundException
* @return Node[]
* @throws \OCP\Files\NotFoundException
*/
public function getDirectoryListing() {
$folderContent = $this->view->getDirectoryContent($this->path);
Expand Down Expand Up @@ -198,6 +207,17 @@ public function newFile($path, $content = null) {
throw new NotPermittedException('No create permission for path');
}

private function queryFromOperator(ISearchOperator $operator, string $uid = null): ISearchQuery {
if ($uid === null) {
$user = null;
} else {
/** @var IUserManager $userManager */
$userManager = \OC::$server->query(IUserManager::class);
$user = $userManager->get($uid);
}
return new SearchQuery($operator, 0, 0, [], $user);
}

/**
* search for files with the name matching $query
*
Expand All @@ -206,45 +226,27 @@ public function newFile($path, $content = null) {
*/
public function search($query) {
if (is_string($query)) {
return $this->searchCommon('search', ['%' . $query . '%']);
} else {
return $this->searchCommon('searchQuery', [$query]);
$query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', '%' . $query . '%'));
}
}

/**
* search for files by mimetype
*
* @param string $mimetype
* @return Node[]
*/
public function searchByMime($mimetype) {
return $this->searchCommon('searchByMime', [$mimetype]);
}

/**
* search for files by tag
*
* @param string|int $tag name or tag id
* @param string $userId owner of the tags
* @return Node[]
*/
public function searchByTag($tag, $userId) {
return $this->searchCommon('searchByTag', [$tag, $userId]);
}

/**
* @param string $method cache method
* @param array $args call args
* @return \OC\Files\Node\Node[]
*/
private function searchCommon($method, $args) {
$limitToHome = ($method === 'searchQuery')? $args[0]->limitToHome(): false;
// Limit+offset for queries with ordering
//
// Because we currently can't do ordering between the results from different storages in sql
// The only way to do ordering is requesting the $limit number of entries from all storages
// sorting them and returning the first $limit entries.
//
// For offset we have the same problem, we don't know how many entries from each storage should be skipped
// by a given $offset, so instead we query $offset + $limit from each storage and return entries $offset..($offset+$limit)
// after merging and sorting them.
//
// This is suboptimal but because limit and offset tend to be fairly small in real world use cases it should
// still be significantly better than disabling paging altogether

$limitToHome = $query->limitToHome();
if ($limitToHome && count(explode('/', $this->path)) !== 3) {
throw new \InvalidArgumentException('searching by owner is only allows on the users home folder');
}

$files = [];
$rootLength = strlen($this->path);
$mount = $this->root->getMount($this->path);
$storage = $mount->getStorage();
Expand All @@ -253,45 +255,106 @@ private function searchCommon($method, $args) {
if ($internalPath !== '') {
$internalPath = $internalPath . '/';
}
$internalRootLength = strlen($internalPath);

$subQueryLimit = $query->getLimit() > 0 ? $query->getLimit() + $query->getOffset() : 0;
$rootQuery = new SearchQuery(
new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
new SearchComparison(ISearchComparison::COMPARE_LIKE, 'path', $internalPath . '%'),
$query->getSearchOperation(),
]
),
$subQueryLimit,
0,
$query->getOrder(),
$query->getUser()
);

$files = [];

$cache = $storage->getCache('');

$results = call_user_func_array([$cache, $method], $args);
$results = $cache->searchQuery($rootQuery);
foreach ($results as $result) {
if ($internalRootLength === 0 or substr($result['path'], 0, $internalRootLength) === $internalPath) {
$result['internalPath'] = $result['path'];
$result['path'] = substr($result['path'], $internalRootLength);
$result['storage'] = $storage;
$files[] = new \OC\Files\FileInfo($this->path . '/' . $result['path'], $storage, $result['internalPath'], $result, $mount);
}
$files[] = $this->cacheEntryToFileInfo($mount, '', $internalPath, $result);
}

if (!$limitToHome) {
$mounts = $this->root->getMountsIn($this->path);
foreach ($mounts as $mount) {
$subQuery = new SearchQuery(
$query->getSearchOperation(),
$subQueryLimit,
0,
$query->getOrder(),
$query->getUser()
);

$storage = $mount->getStorage();
if ($storage) {
$cache = $storage->getCache('');

$relativeMountPoint = ltrim(substr($mount->getMountPoint(), $rootLength), '/');
$results = call_user_func_array([$cache, $method], $args);
$results = $cache->searchQuery($subQuery);
foreach ($results as $result) {
$result['internalPath'] = $result['path'];
$result['path'] = $relativeMountPoint . $result['path'];
$result['storage'] = $storage;
$files[] = new \OC\Files\FileInfo($this->path . '/' . $result['path'], $storage,
$result['internalPath'], $result, $mount);
$files[] = $this->cacheEntryToFileInfo($mount, $relativeMountPoint, '', $result);
}
}
}
}

$order = $query->getOrder();
if ($order) {
usort($files, function (FileInfo $a,FileInfo $b) use ($order) {
foreach ($order as $orderField) {
$cmp = $orderField->sortFileInfo($a, $b);
if ($cmp !== 0) {
return $cmp;
}
}
return 0;
});
}
$files = array_values(array_slice($files, $query->getOffset(), $query->getLimit() > 0 ? $query->getLimit() : null));

return array_map(function (FileInfo $file) {
return $this->createNode($file->getPath(), $file);
}, $files);
}

private function cacheEntryToFileInfo(IMountPoint $mount, string $appendRoot, string $trimRoot, ICacheEntry $cacheEntry): FileInfo {
$trimLength = strlen($trimRoot);
$cacheEntry['internalPath'] = $cacheEntry['path'];
$cacheEntry['path'] = $appendRoot . substr($cacheEntry['path'], $trimLength);
return new \OC\Files\FileInfo($this->path . '/' . $cacheEntry['path'], $mount->getStorage(), $cacheEntry['internalPath'], $cacheEntry, $mount);
}

/**
* search for files by mimetype
*
* @param string $mimetype
* @return Node[]
*/
public function searchByMime($mimetype) {
if (strpos($mimetype, '/') === false) {
$query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', $mimetype . '/%'));
} else {
$query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', $mimetype));
}
return $this->search($query);
}

/**
* search for files by tag
*
* @param string|int $tag name or tag id
* @param string $userId owner of the tags
* @return Node[]
*/
public function searchByTag($tag, $userId) {
$query = $this->queryFromOperator(new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'tagname', $tag), $userId);
return $this->search($query);
}

/**
* @param int $id
* @return \OC\Files\Node\Node[]
Expand All @@ -318,7 +381,7 @@ public function getById($id) {

if (count($mountsContainingFile) === 0) {
if ($user === $this->getAppDataDirectoryName()) {
return $this->getByIdInRootMount((int) $id);
return $this->getByIdInRootMount((int)$id);
}
return [];
}
Expand Down Expand Up @@ -381,11 +444,11 @@ protected function getByIdInRootMount(int $id): array {

return [$this->root->createNode(
$absolutePath, new \OC\Files\FileInfo(
$absolutePath,
$mount->getStorage(),
$cacheEntry->getPath(),
$cacheEntry,
$mount
$absolutePath,
$mount->getStorage(),
$cacheEntry->getPath(),
$cacheEntry,
$mount
))];
}

Expand Down
25 changes: 25 additions & 0 deletions lib/private/Files/Search/SearchOrder.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

namespace OC\Files\Search;

use OCP\Files\FileInfo;
use OCP\Files\Search\ISearchOrder;

class SearchOrder implements ISearchOrder {
Expand Down Expand Up @@ -55,4 +56,28 @@ public function getDirection() {
public function getField() {
return $this->field;
}

public function sortFileInfo(FileInfo $a, FileInfo $b): int {
$cmp = $this->sortFileInfoNoDirection($a, $b);
return $cmp * ($this->direction === ISearchOrder::DIRECTION_ASCENDING ? 1 : -1);
}

private function sortFileInfoNoDirection(FileInfo $a, FileInfo $b): int {
switch ($this->field) {
case 'name':
return $a->getName() <=> $b->getName();
case 'mimetype':
return $a->getMimetype() <=> $b->getMimetype();
case 'mtime':
return $a->getMtime() <=> $b->getMtime();
case 'size':
return $a->getSize() <=> $b->getSize();
case 'fileid':
return $a->getId() <=> $b->getId();
case 'permissions':
return $a->getPermissions() <=> $b->getPermissions();
default:
return 0;
}
}
}
8 changes: 4 additions & 4 deletions lib/private/Files/Search/SearchQuery.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class SearchQuery implements ISearchQuery {
private $offset;
/** @var ISearchOrder[] */
private $order;
/** @var IUser */
/** @var ?IUser */
private $user;
private $limitToHome;

Expand All @@ -48,15 +48,15 @@ class SearchQuery implements ISearchQuery {
* @param int $limit
* @param int $offset
* @param array $order
* @param IUser $user
* @param ?IUser $user
* @param bool $limitToHome
*/
public function __construct(
ISearchOperator $searchOperation,
int $limit,
int $offset,
array $order,
IUser $user,
?IUser $user = null,
bool $limitToHome = false
) {
$this->searchOperation = $searchOperation;
Expand Down Expand Up @@ -96,7 +96,7 @@ public function getOrder() {
}

/**
* @return IUser
* @return ?IUser
*/
public function getUser() {
return $this->user;
Expand Down
Loading

0 comments on commit 2814fc6

Please sign in to comment.