From 010473ca1fab51f5df3cccc51dd42b0271b0e867 Mon Sep 17 00:00:00 2001 From: Daniel Kesselberg Date: Fri, 28 Sep 2018 22:59:34 +0200 Subject: [PATCH 1/6] Add method to check if directory exists Signed-off-by: Daniel Kesselberg --- .../lib/Lib/Storage/AmazonS3.php | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index ea7ca42dfefbc..403dddc9106df 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -60,6 +60,8 @@ public function needsPartFile() { /** @var CappedMemoryCache|Result[] */ private $objectCache; + /** @var CappedMemoryCache|bool[] */ + private $directoryCache; /** @var CappedMemoryCache|array */ private $filesCache; @@ -68,6 +70,7 @@ public function __construct($parameters) { parent::__construct($parameters); $this->parseParams($parameters); $this->objectCache = new CappedMemoryCache(); + $this->directoryCache = new CappedMemoryCache(); $this->filesCache = new CappedMemoryCache(); } @@ -98,6 +101,7 @@ private function cleanKey($path) { private function clearCache() { $this->objectCache = new CappedMemoryCache(); + $this->directoryCache = new CappedMemoryCache(); $this->filesCache = new CappedMemoryCache(); } @@ -135,6 +139,41 @@ private function headObject($key) { return $this->objectCache[$key]; } + /** + * Return true if directory exists + * + * There are no folders in s3. A folder like structure could be archived + * by prefixing files with the folder name. + * + * Implementation from flysystem-aws-s3-v3: + * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694 + * + * @param $location + * @return bool + * @throws \Exception + */ + protected function doesDirectoryExist($location) { + if (!isset($this->directoryCache[$location])) { + // Maybe this isn't an actual key, but a prefix. + // Do a prefix listing of objects to determine. + try { + $result = $this->getConnection()->listObjects([ + 'Bucket' => $this->bucket, + 'Prefix' => rtrim($location, '/') . '/', + 'MaxKeys' => 1, + ]); + $this->directoryCache[$location] = $result['Contents'] || $result['CommonPrefixes']; + } catch (S3Exception $e) { + if ($e->getStatusCode() === 403) { + $this->directoryCache[$location] = false; + } + throw $e; + } + } + + return $this->directoryCache[$location]; + } + /** * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name. * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home @@ -393,7 +432,7 @@ private function getLastModified($path) { public function is_dir($path) { $path = $this->normalizePath($path); try { - return $this->isRoot($path) || $this->headObject($path . '/'); + return $this->isRoot($path) || $this->doesDirectoryExist($path); } catch (S3Exception $e) { \OC::$server->getLogger()->logException($e, ['app' => 'files_external']); return false; @@ -411,7 +450,7 @@ public function filetype($path) { if (isset($this->filesCache[$path]) || $this->headObject($path)) { return 'file'; } - if ($this->headObject($path . '/')) { + if ($this->doesDirectoryExist($path)) { return 'dir'; } } catch (S3Exception $e) { From 963d35c78c80181591a314cb89b58448fa17de9e Mon Sep 17 00:00:00 2001 From: Daniel Kesselberg Date: Sat, 29 Sep 2018 21:10:40 +0200 Subject: [PATCH 2/6] Rename $location to $path Signed-off-by: Daniel Kesselberg --- apps/files_external/lib/Lib/Storage/AmazonS3.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index 403dddc9106df..c935d500200be 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -148,30 +148,30 @@ private function headObject($key) { * Implementation from flysystem-aws-s3-v3: * https://github.com/thephpleague/flysystem-aws-s3-v3/blob/8241e9cc5b28f981e0d24cdaf9867f14c7498ae4/src/AwsS3Adapter.php#L670-L694 * - * @param $location + * @param $path * @return bool * @throws \Exception */ - protected function doesDirectoryExist($location) { - if (!isset($this->directoryCache[$location])) { + protected function doesDirectoryExist($path) { + if (!isset($this->directoryCache[$path])) { // Maybe this isn't an actual key, but a prefix. // Do a prefix listing of objects to determine. try { $result = $this->getConnection()->listObjects([ 'Bucket' => $this->bucket, - 'Prefix' => rtrim($location, '/') . '/', + 'Prefix' => rtrim($path, '/') . '/', 'MaxKeys' => 1, ]); - $this->directoryCache[$location] = $result['Contents'] || $result['CommonPrefixes']; + $this->directoryCache[$path] = $result['Contents'] || $result['CommonPrefixes']; } catch (S3Exception $e) { if ($e->getStatusCode() === 403) { - $this->directoryCache[$location] = false; + $this->directoryCache[$path] = false; } throw $e; } } - return $this->directoryCache[$location]; + return $this->directoryCache[$path]; } /** From 12863e0d243bb72057f1a005a0b319910f4da8c5 Mon Sep 17 00:00:00 2001 From: Daniel Kesselberg Date: Sat, 29 Sep 2018 21:12:39 +0200 Subject: [PATCH 3/6] Change visibility to private (from protected) Signed-off-by: Daniel Kesselberg --- apps/files_external/lib/Lib/Storage/AmazonS3.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index c935d500200be..b48c3fbfd8357 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -152,7 +152,7 @@ private function headObject($key) { * @return bool * @throws \Exception */ - protected function doesDirectoryExist($path) { + private function doesDirectoryExist($path) { if (!isset($this->directoryCache[$path])) { // Maybe this isn't an actual key, but a prefix. // Do a prefix listing of objects to determine. From 80340a8d463e2d115ed1c33c0f0f54542f3e952e Mon Sep 17 00:00:00 2001 From: Daniel Kesselberg Date: Tue, 1 Jan 2019 18:15:07 +0100 Subject: [PATCH 4/6] Fill directory cache from opendir Signed-off-by: Daniel Kesselberg --- apps/files_external/lib/Lib/Storage/AmazonS3.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index b48c3fbfd8357..019bfa17b3c71 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -60,6 +60,7 @@ public function needsPartFile() { /** @var CappedMemoryCache|Result[] */ private $objectCache; + /** @var CappedMemoryCache|bool[] */ private $directoryCache; @@ -114,7 +115,7 @@ private function invalidateCache($key) { unset($this->objectCache[$existingKey]); } } - unset($this->filesCache[$key]); + unset($this->directoryCache[$key], $this->filesCache[$key]); } /** @@ -334,6 +335,7 @@ public function opendir($path) { if (is_array($result['CommonPrefixes'])) { foreach ($result['CommonPrefixes'] as $prefix) { $files[] = substr(trim($prefix['Prefix'], '/'), strlen($path)); + $this->directoryCache[substr(trim($prefix['Prefix'], '/'), strlen($path))] = true; } } if (is_array($result['Contents'])) { From 75b89440612ff45b55dabcb24c2932ea37ee2de0 Mon Sep 17 00:00:00 2001 From: Daniel Kesselberg Date: Thu, 14 Feb 2019 18:06:09 +0100 Subject: [PATCH 5/6] Don't strip path from directory prefix Signed-off-by: Daniel Kesselberg --- apps/files_external/lib/Lib/Storage/AmazonS3.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index 019bfa17b3c71..befa65671004f 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -335,7 +335,7 @@ public function opendir($path) { if (is_array($result['CommonPrefixes'])) { foreach ($result['CommonPrefixes'] as $prefix) { $files[] = substr(trim($prefix['Prefix'], '/'), strlen($path)); - $this->directoryCache[substr(trim($prefix['Prefix'], '/'), strlen($path))] = true; + $this->directoryCache[trim($prefix['Prefix'], '/')] = true; } } if (is_array($result['Contents'])) { @@ -433,6 +433,11 @@ private function getLastModified($path) { public function is_dir($path) { $path = $this->normalizePath($path); + + if (isset($this->filesCache[$path])) { + return false; + } + try { return $this->isRoot($path) || $this->doesDirectoryExist($path); } catch (S3Exception $e) { From 72d22a48284dd9e700d3586db65cca50e9d6f2f0 Mon Sep 17 00:00:00 2001 From: Daniel Kesselberg Date: Sat, 30 Mar 2019 12:27:52 +0100 Subject: [PATCH 6/6] Use local variable for directory name Signed-off-by: Daniel Kesselberg --- apps/files_external/lib/Lib/Storage/AmazonS3.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/files_external/lib/Lib/Storage/AmazonS3.php b/apps/files_external/lib/Lib/Storage/AmazonS3.php index befa65671004f..c6cd6e1b2ca68 100644 --- a/apps/files_external/lib/Lib/Storage/AmazonS3.php +++ b/apps/files_external/lib/Lib/Storage/AmazonS3.php @@ -334,8 +334,9 @@ public function opendir($path) { // sub folders if (is_array($result['CommonPrefixes'])) { foreach ($result['CommonPrefixes'] as $prefix) { - $files[] = substr(trim($prefix['Prefix'], '/'), strlen($path)); - $this->directoryCache[trim($prefix['Prefix'], '/')] = true; + $directoryName = trim($prefix['Prefix'], '/'); + $files[] = substr($directoryName, strlen($path)); + $this->directoryCache[$directoryName] = true; } } if (is_array($result['Contents'])) {