Adding repair step for sub share

Adding repair step for sub share. When share table due to some magic gets duplicate entries ( except the id or primary key ), then this repair step comes for rescue. Signed-off-by: Sujith H <sharidasan@owncloud.com>
owncloud · Jan 23, 2018 · f3ff72a · f3ff72a
1 parent e1c4425
commit f3ff72a
Show file tree

Hide file tree

Showing 3 changed files with 300 additions and 0 deletions.
diff --git a/lib/private/Repair.php b/lib/private/Repair.php
@@ -37,6 +37,7 @@
 use OC\Repair\OldGroupMembershipShares;
 use OC\Repair\RemoveGetETagEntries;
 use OC\Repair\RemoveRootShares;
+use OC\Repair\RepairSubShares;
 use OC\Repair\SharePropagation;
 use OC\Repair\SqliteAutoincrement;
 use OC\Repair\DropOldTables;
@@ -157,6 +158,9 @@ public static function getRepairSteps() {
 				\OC::$server->getConfig(),
 				\OC::$server->getAppConfig()
 			),
+			new RepairSubShares(
+				\OC::$server->getDatabaseConnection()
+			),
 		];
 	}
 

diff --git a/lib/private/Repair/RepairSubShares.php b/lib/private/Repair/RepairSubShares.php
@@ -0,0 +1,105 @@
+<?php
+/**
+ * @author Sujith Haridasan <sharidasan@owncloud.com>
+ *
+ * @copyright Copyright (c) 2018, ownCloud GmbH
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Repair;
+
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IConfig;
+use OCP\IDBConnection;
+use OCP\Migration\IOutput;
+use OCP\Migration\IRepairStep;
+
+class RepairSubShares implements IRepairStep {
+
+	/** @var IDBConnection  */
+	private $connection;
+
+	/** @var  IQueryBuilder */
+	private $getDuplicateRows;
+
+	/** @var  IQueryBuilder */
+	private $deleteShareId;
+
+	public function __construct(
+		IDBConnection $connection) {
+		$this->connection = $connection;
+	}
+
+	public function getName() {
+		return "Repair sub shares";
+	}
+
+	/**
+	 * Set query to remove duplicate rows.
+	 * i.e, except id all columns are same for oc_share
+	 * Also set query to select rows which have duplicate rows of share.
+	 */
+	private function setRemoveAndSelectQuery() {
+		/**
+		 * Retrieves the duplicate rows with different id's
+		 * of oc_share
+		 */
+		$builder = $this->connection->getQueryBuilder();
+		$builder
+			->select('id', 'parent', $builder->createFunction('count(*)'))
+			->from('share')
+			->where($builder->expr()->eq('share_type', $builder->createNamedParameter(2)))
+			->groupBy('parent')
+			->addGroupBy('id')
+			->addGroupBy('share_with')
+			->having('count(*) > 1')->setMaxResults(1000);
+
+		$this->getDuplicateRows = $builder;
+
+		$builder = $this->connection->getQueryBuilder();
+		$builder
+			->delete('share')
+			->where($builder->expr()->eq('id', $builder->createParameter('shareId')));
+
+		$this->deleteShareId = $builder;
+	}
+
+	public function run(IOutput $output) {
+		$deletedEntries = 0;
+		$this->setRemoveAndSelectQuery();
+
+		/**
+		 * Going for pagination because if there are 1 million rows
+		 * it wont be easy to scale the data
+		 */
+		do {
+			$results = $this->getDuplicateRows->execute();
+			$rows = $results->fetchAll();
+			$results->closeCursor();
+			$lastResultCount = 0;
+
+			foreach ($rows as $row) {
+				$deletedEntries += $this->deleteShareId->setParameter('shareId', (int) $row['id'])
+					->execute();
+				$lastResultCount++;
+			}
+		} while($lastResultCount > 0);
+
+		if ($deletedEntries > 0) {
+			$output->info('Removed ' . $deletedEntries . ' shares where duplicate rows where found');
+		}
+	}
+}
diff --git a/tests/lib/Repair/RepairSubSharesTest.php b/tests/lib/Repair/RepairSubSharesTest.php
@@ -0,0 +1,191 @@
+<?php
+/**
+ * @author Sujith Haridasan <sharidasan@owncloud.com>
+ *
+ * @copyright Copyright (c) 2018, ownCloud GmbH
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace Test\Repair;
+
+use OC\Repair\RepairSubShares;
+use OCP\Migration\IOutput;
+use OCP\Migration\IRepairStep;
+use Test\TestCase;
+use Test\Traits\UserTrait;
+
+/**
+ * Test for repairing invalid sub shares
+ *
+ * @group  DB
+ *
+ * @see \OC\Repair\RepairSubShares
+ * @package Test\Repair
+ */
+class RepairSubSharesTest extends TestCase {
+	use UserTrait;
+
+	/** @var  \OCP\IDBConnection */
+	private $connection;
+
+	/** @var  IRepairStep */
+	private $repair;
+	protected function setUp() {
+		parent::setUp();
+
+		$this->connection = \OC::$server->getDatabaseConnection();
+		$this->repair = new RepairSubShares($this->connection);
+		$this->createUser('admin');
+	}
+
+	protected function tearDown() {
+		$this->deleteAllUsersAndGroups();
+		$this->deleteAllShares();
+		parent::tearDown();
+	}
+
+	public function deleteAllUsersAndGroups() {
+		$this->tearDownUserTrait();
+		$qb = $this->connection->getQueryBuilder();
+		$qb->delete('groups')->execute();
+		$qb->delete('group_user')->execute();
+	}
+
+	public function deleteAllShares() {
+		$qb = $this->connection->getQueryBuilder();
+		$qb->delete('share')->execute();
+	}
+
+	/**
+	 * This is a very basic test
+	 * This test would populate DB with data
+	 * and later, remove the duplicates to test
+	 * if the step is working properly
+	 */
+	public function testPopulateDBAndRemoveDuplicates() {
+
+		$qb = $this->connection->getQueryBuilder();
+		//Create 10 users and 3 groups.
+		//add 3 users to each group
+		$userName = "user";
+		$groupName = "group";
+		$folderName = "/test";
+		$time = time();
+		$groupCount = 1;
+		$totalGroups = 3;
+		for($userCount = 1; $userCount <= 10; $userCount++) {
+			$user = $this->createUser($userName.$userCount);
+			if (\OC::$server->getGroupManager()->groupExists($groupName.$groupCount) === false) {
+				\OC::$server->getGroupManager()->createGroup($groupName.$groupCount);
+			}
+			\OC::$server->getGroupManager()->get($groupName.$groupCount)->addUser($user);
+
+			//Create a group share
+			$qb->insert('share')
+				->values([
+					'share_type' => $qb->expr()->literal('2'),
+					'share_with' => $qb->expr()->literal($userName.$groupCount),
+					'uid_owner' => $qb->expr()->literal('admin'),
+					'uid_initiator' => $qb->expr()->literal('admin'),
+					'parent' => $qb->expr()->literal(1),
+					'item_type' => $qb->expr()->literal('folder'),
+					'item_source' => $qb->expr()->literal(24),
+					'file_source' => $qb->expr()->literal(24),
+					'file_target' => $qb->expr()->literal($folderName.$groupCount),
+					'permissions' => $qb->expr()->literal(31),
+					'stime' => $qb->expr()->literal($time),
+				])
+				->execute();
+
+			/**
+			 * Group count incremented once value of userCount reaches multiple of 3
+			 */
+			if (($userCount % $totalGroups) === 0) {
+				$groupCount++;
+				$time = time();
+			}
+		}
+
+		$outputMock = $this->createMock(IOutput::class);
+		$this->repair->run($outputMock);
+
+		$qb = $this->connection->getQueryBuilder();
+		$qb->select('id', 'parent', $qb->createFunction('count(*)'))
+			->from('share')
+			->where($qb->expr()->eq('share_type', $qb->createNamedParameter(2)))
+			->groupBy('parent')
+			->addGroupBy('id')
+			->addGroupBy('share_with')
+			->having('count(*) > 1')->setMaxResults(1000);
+
+		$results = $qb->execute()->fetchAll();
+		$this->assertCount(0, $results);
+	}
+
+	/**
+	 * This is to test large rows i.e, greater than 2000
+	 * with duplicates
+	 */
+	public function testLargeDuplicateShareRows() {
+		$qb = $this->connection->getQueryBuilder();
+		$userName = "user";
+		$time = time();
+		$groupCount = 0;
+		$folderName = "/test";
+		$maxUsersPerGroup = 1000;
+		$parent = $groupCount + 1;
+		for ($userCount = 0; $userCount < 5500; $userCount++) {
+			/**
+			 * groupCount is incremented once userCount reaches
+			 * multiple of maxUsersPerGroup.
+			 */
+			if (($userCount % $maxUsersPerGroup) === 0) {
+				$groupCount++;
+				$parent = $groupCount;
+			}
+			$qb->insert('share')
+				->values([
+					'share_type' => $qb->expr()->literal('2'),
+					'share_with' => $qb->expr()->literal($userName.$groupCount),
+					'uid_owner' => $qb->expr()->literal('admin'),
+					'uid_initiator' => $qb->expr()->literal('admin'),
+					'parent' => $qb->expr()->literal($parent),
+					'item_type' => $qb->expr()->literal('folder'),
+					'item_source' => $qb->expr()->literal(24),
+					'file_source' => $qb->expr()->literal(24),
+					'file_target' => $qb->expr()->literal($folderName.$groupCount),
+					'permissions' => $qb->expr()->literal(31),
+					'stime' => $qb->expr()->literal($time),
+				])
+				->execute();
+		}
+
+		$outputMock = $this->createMock(IOutput::class);
+		$this->repair->run($outputMock);
+
+		$qb = $this->connection->getQueryBuilder();
+		$qb->select('id', 'parent', $qb->createFunction('count(*)'))
+			->from('share')
+			->where($qb->expr()->eq('share_type', $qb->createNamedParameter(2)))
+			->groupBy('parent')
+			->addGroupBy('id')
+			->addGroupBy('share_with')
+			->having('count(*) > 1')->setMaxResults(1000);
+
+		$results = $qb->execute()->fetchAll();
+		$this->assertCount(0, $results);
+	}
+}