Catch duplicate insertion errors while scanning files

When two scanning processed run at the same time, for example when
scan.php and list.php were called at the same time on an external
storage mountpoint, duplicate insertion errors can occurs.

These errors are now logged and ignored.

Since both scans are running in parallel transactions, they don't see
each other's changes directly in the DB which can cause duplicate
insertion errors for mime types as well, but those mime types can't be
selected yet. The solution to this is to force-reload the mimetypes list
after the transaction is finished.

Fixes #5199

lib/private/files/cache/cache.php

@@ -69,9 +69,15 @@ public function getMimetypeId($mime) {
 		}
 
 		if (!isset(self::$mimetypeIds[$mime])) {
-			$result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
-			self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
-			self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
+			try{
+				$result = \OC_DB::executeAudited('INSERT INTO `*PREFIX*mimetypes`(`mimetype`) VALUES(?)', array($mime));
+				self::$mimetypeIds[$mime] = \OC_DB::insertid('*PREFIX*mimetypes');
+				self::$mimetypes[self::$mimetypeIds[$mime]] = $mime;
+			}
+			catch (\Doctrine\DBAL\DBALException $e){
+				\OC_Log::write('core', 'Exception during mimetype insertion: ' . $e->getmessage(), \OC_Log::DEBUG);
+				return -1;
+			}
 		} 
 
 		return self::$mimetypeIds[$mime];
@@ -84,8 +90,8 @@ public function getMimetype($id) {
 
 		return isset(self::$mimetypes[$id]) ? self::$mimetypes[$id] : null;
 	}
-	
-	protected function loadMimetypes(){
+
+	public function loadMimetypes(){
 			$result = \OC_DB::executeAudited('SELECT `id`, `mimetype` FROM `*PREFIX*mimetypes`', array());
 			if ($result) {
 				while ($row = $result->fetchRow()) {

lib/private/files/cache/scanner.php

@@ -190,24 +190,34 @@ public function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse =
 		}
 		$newChildren = array();
 		if ($this->storage->is_dir($path) && ($dh = $this->storage->opendir($path))) {
+			$exceptionOccurred = false;
 			\OC_DB::beginTransaction();
 			if (is_resource($dh)) {
 				while (($file = readdir($dh)) !== false) {
 					$child = ($path) ? $path . '/' . $file : $file;
 					if (!Filesystem::isIgnoredDir($file)) {
 						$newChildren[] = $file;
-						$data = $this->scanFile($child, $reuse, true);
-						if ($data) {
-							if ($data['size'] === -1) {
-								if ($recursive === self::SCAN_RECURSIVE) {
-									$childQueue[] = $child;
-								} else {
-									$size = -1;
+						try {
+							$data = $this->scanFile($child, $reuse, true);
+							if ($data) {
+								if ($data['size'] === -1) {
+									if ($recursive === self::SCAN_RECURSIVE) {
+										$childQueue[] = $child;
+									} else {
+										$size = -1;
+									}
+								} else if ($size !== -1) {
+									$size += $data['size'];
 								}
-							} else if ($size !== -1) {
-								$size += $data['size'];
 							}
 						}
+						catch (\Doctrine\DBAL\DBALException $ex){
+							// might happen if inserting duplicate while a scanning
+							// process is running in parallel
+							// log and ignore
+							\OC_Log::write('core', 'Exception while scanning file "' . $child . '": ' . $ex->getMessage(), \OC_Log::DEBUG);
+							$exceptionOccurred = true;
+						}
 					}
 				}
 			}
@@ -217,6 +227,14 @@ public function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse =
 				$this->cache->remove($child);
 			}
 			\OC_DB::commit();
+			if ($exceptionOccurred){
+				// It might happen that the parallel scan process has already
+				// inserted mimetypes but those weren't available yet inside the transaction
+				// To make sure to have the updated mime types in such cases,
+				// we reload them here
+				$this->cache->loadMimetypes();
+			}
+
 			foreach ($childQueue as $child) {
 				$childSize = $this->scanChildren($child, self::SCAN_RECURSIVE, $reuse);
 				if ($childSize === -1) {