Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix directory detection for s3 #14913

Merged
merged 6 commits into from
Sep 26, 2019
Merged

Fix directory detection for s3 #14913

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
010473c
Add method to check if directory exists
kesselb Sep 28, 2018
963d35c
Rename $location to $path
kesselb Sep 29, 2018
12863e0
Change visibility to private (from protected)
kesselb Sep 29, 2018
80340a8
Fill directory cache from opendir
kesselb Jan 1, 2019
75b8944
Don't strip path from directory prefix
kesselb Feb 14, 2019
72d22a4
Use local variable for directory name
kesselb Mar 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 51 additions & 4 deletions apps/files_external/lib/Lib/Storage/AmazonS3.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,13 +61,17 @@ public function needsPartFile() {
/** @var CappedMemoryCache|Result[] */
private $objectCache;

/** @var CappedMemoryCache|bool[] */
private $directoryCache;

/** @var CappedMemoryCache|array */
private $filesCache;

public function __construct($parameters) {
parent::__construct($parameters);
$this->parseParams($parameters);
$this->objectCache = new CappedMemoryCache();
$this->directoryCache = new CappedMemoryCache();
$this->filesCache = new CappedMemoryCache();
}

Expand Down Expand Up @@ -98,6 +102,7 @@ private function cleanKey($path) {

private function clearCache() {
$this->objectCache = new CappedMemoryCache();
$this->directoryCache = new CappedMemoryCache();
$this->filesCache = new CappedMemoryCache();
}

Expand All @@ -110,7 +115,7 @@ private function invalidateCache($key) {
unset($this->objectCache[$existingKey]);
}
}
unset($this->filesCache[$key]);
unset($this->directoryCache[$key], $this->filesCache[$key]);
}

/**
Expand All @@ -135,6 +140,41 @@ private function headObject($key) {
return $this->objectCache[$key];
}

/**
* Return true if directory exists
*
* There are no folders in s3. A folder like structure could be archived
* by prefixing files with the folder name.
*
* Implementation from flysystem-aws-s3-v3:
* https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694
*
* @param $path
* @return bool
* @throws \Exception
*/
private function doesDirectoryExist($path) {
if (!isset($this->directoryCache[$path])) {
// Maybe this isn't an actual key, but a prefix.
// Do a prefix listing of objects to determine.
try {
$result = $this->getConnection()->listObjects([
'Bucket' => $this->bucket,
'Prefix' => rtrim($path, '/') . '/',
'MaxKeys' => 1,
]);
$this->directoryCache[$path] = $result['Contents'] || $result['CommonPrefixes'];
} catch (S3Exception $e) {
if ($e->getStatusCode() === 403) {
$this->directoryCache[$path] = false;
}
throw $e;
}
}

return $this->directoryCache[$path];
}

/**
* Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
* TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
Expand Down Expand Up @@ -294,7 +334,9 @@ public function opendir($path) {
// sub folders
if (is_array($result['CommonPrefixes'])) {
foreach ($result['CommonPrefixes'] as $prefix) {
$files[] = substr(trim($prefix['Prefix'], '/'), strlen($path));
$directoryName = trim($prefix['Prefix'], '/');
$files[] = substr($directoryName, strlen($path));
$this->directoryCache[$directoryName] = true;
}
}
if (is_array($result['Contents'])) {
Expand Down Expand Up @@ -392,8 +434,13 @@ private function getLastModified($path) {

public function is_dir($path) {
$path = $this->normalizePath($path);

if (isset($this->filesCache[$path])) {
return false;
}

try {
return $this->isRoot($path) || $this->headObject($path . '/');
return $this->isRoot($path) || $this->doesDirectoryExist($path);
} catch (S3Exception $e) {
\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
return false;
Expand All @@ -411,7 +458,7 @@ public function filetype($path) {
if (isset($this->filesCache[$path]) || $this->headObject($path)) {
return 'file';
}
if ($this->headObject($path . '/')) {
if ($this->doesDirectoryExist($path)) {
return 'dir';
}
} catch (S3Exception $e) {
Expand Down