perf: Partition preview migration job

core/BackgroundJobs/PreviewMigrationJob.php

@@ -11,100 +11,134 @@
 
 use OC\Preview\PreviewMigrationService;
 use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\IJobList;
 use OCP\BackgroundJob\TimedJob;
-use OCP\Files\FileInfo;
 use OCP\Files\IRootFolder;
 use OCP\IAppConfig;
 use OCP\IConfig;
 use Override;
 use Psr\Log\LoggerInterface;
 
 class PreviewMigrationJob extends TimedJob {
+	public const int PARTITIONS = 16;
+	private const int BATCH_SIZE = 500;
 	private string $previewRootPath;
 
 	public function __construct(
 		ITimeFactory $time,
 		private readonly IAppConfig $appConfig,
-		private readonly IConfig $config,
+		IConfig $config,
 		private readonly IRootFolder $rootFolder,
 		private readonly PreviewMigrationService $migrationService,
+		private readonly IJobList $jobList,
 		private readonly LoggerInterface $logger,
 	) {
 		parent::__construct($time);
-
 		$this->setTimeSensitivity(self::TIME_INSENSITIVE);
-		$this->setInterval(24 * 60 * 60);
-		$this->previewRootPath = 'appdata_' . $this->config->getSystemValueString('instanceid') . '/preview/';
+		$this->setInterval(62 * 60); // every 62 min as the job takes 60 min
+		$this->previewRootPath = 'appdata_' . $config->getSystemValueString('instanceid') . '/preview/';
 	}
 
 	#[Override]
 	protected function run(mixed $argument): void {
 		if ($this->appConfig->getValueBool('core', 'previewMovedDone')) {
+			$this->jobList->removeById($this->getId());
+			return;
+		}
+
+		$partition = (int)($argument['partition'] ?? 0);
+		if ($partition < 0 || $partition >= self::PARTITIONS) {
+			$this->jobList->removeById($this->getId());
 			return;
 		}
 
+		if (!$this->runPartition($partition)) {
+			return;
+		}
+
+		$this->appConfig->setValueBool('core', 'previewMigrationPartition' . $partition, true);
+		$this->jobList->removeById($this->getId());
+		for ($i = 0; $i < self::PARTITIONS; $i++) {
+			if (!$this->appConfig->getValueBool('core', 'previewMigrationPartition' . $i, false)) {
+				return;
+			}
+		}
+
+		$this->appConfig->setValueBool('core', 'previewMovedDone', true);
+	}
+
+	private function runPartition(int $partition): bool {
 		$storage = $this->rootFolder->getMountPoint()->getStorage();
 		if ($storage === null) {
 			$this->logger->warning('Preview migration skipped: the root mount point has no storage.');
-			$this->appConfig->setValueBool('core', 'previewMovedDone', true);
-			return;
+			return true;
 		}
 
-		$cache = $storage->getCache();
-		$previewRootId = $cache->getId(rtrim($this->previewRootPath, '/'));
+		$previewRootId = $storage->getCache()->getId(rtrim($this->previewRootPath, '/'));
 		if ($previewRootId === -1) {
-			// No previews were ever generated, or the storage config no longer
-			// matches the one the filecache data was recorded under.
 			$this->logger->warning('Preview migration skipped: no preview root found at "{path}" on storage "{storageId}".', [
 				'path' => $this->previewRootPath,
 				'storageId' => $storage->getId(),
 			]);
-			$this->appConfig->setValueBool('core', 'previewMovedDone', true);
-			return;
+			return true;
 		}
 
 		$startTime = time();
-
-		// Walk the preview folder tree via the `parent` column, which is indexed on
-		// every supported database platform.
-		//
-		// Depth from the preview root tells us which structure a leaf folder holds:
-		// - depth 1: legacy flat structure, e.g. preview/<fileid>/<size>.png
-		// - depth 8: hierarchical structure, e.g. preview/a/b/c/d/e/f/g/<fileid>/<size>.png
-		$foldersToVisit = [[$previewRootId, '', 0]];
+		$foldersToVisit = [['id' => $previewRootId, 'name' => '', 'depth' => 0]];
+		$foldersToMigrate = [];
+		// Folders without preview files, by depth; removed once their children are gone.
+		$emptyFolders = [];
 
 		while ($foldersToVisit !== []) {
-			[$folderId, $folderName, $depth] = array_pop($foldersToVisit);
-
-			// Collect the actual preview files here so migrateFileId() doesn't need to
-			// list this folder's contents a second time.
-			$previewEntries = [];
-			foreach ($cache->getFolderContentsById($folderId) as $entry) {
-				if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
-					$foldersToVisit[] = [$entry->getId(), $entry->getName(), $depth + 1];
-				} else {
-					$previewEntries[] = $entry;
+			$folders = array_filter(
+				array_splice($foldersToVisit, -self::BATCH_SIZE),
+				fn (array $folder): bool => $folder['depth'] !== 1 || $this->belongsToPartition($folder['name'], $partition),
+			);
+			$children = $this->migrationService->getFolderChildren(array_column($folders, 'id'));
+
+			foreach ($folders as $folder) {
+				foreach ($children[$folder['id']]['folders'] as $subFolder) {
+					$foldersToVisit[] = [...$subFolder, 'depth' => $folder['depth'] + 1];
 				}
-			}
 
-			if ($previewEntries === [] || !ctype_digit($folderName)) {
-				continue;
+				$files = $children[$folder['id']]['files'];
+				if ($files !== [] && ctype_digit($folder['name'])) {
+					$foldersToMigrate[] = [
+						'fileId' => (int)$folder['name'],
+						'folderId' => $folder['id'],
+						'flat' => $folder['depth'] === 1,
+						'entries' => $files,
+					];
+				} elseif ($folder['depth'] > 0) {
+					$emptyFolders[$folder['depth']][] = $folder['id'];
+				}
 			}
 
-			try {
-				$this->migrationService->migrateFileId((int)$folderName, flatPath: $depth === 1, entries: $previewEntries);
-			} catch (\Exception $e) {
-				$this->logger->error('Failed to migrate preview with fileId: ' . $folderName, [
-					'exception' => $e,
-				]);
-			}
+			if (count($foldersToMigrate) >= self::BATCH_SIZE) {
+				$this->migrationService->migrateFolders($foldersToMigrate);
+				$foldersToMigrate = [];
 
-			// Stop if execution time is more than one hour.
-			if (time() - $startTime > 3600) {
-				return;
+				if (time() - $startTime > 3600) {
+					return false;
+				}
 			}
 		}
 
-		$this->appConfig->setValueBool('core', 'previewMovedDone', true);
+		$this->migrationService->migrateFolders($foldersToMigrate);
+		$this->migrationService->deleteEmptyFolders($emptyFolders);
+
+		return true;
+	}
+
+	private function belongsToPartition(string $folderName, int $partition): bool {
+		if (ctype_digit($folderName)) {
+			return ((int)$folderName % self::PARTITIONS) === $partition;
+		}
+
+		if (strlen($folderName) === 1 && ctype_xdigit($folderName)) {
+			return (hexdec($folderName) % self::PARTITIONS) === $partition;
+		}
+
+		return $partition === 0;
 	}
 }

lib/private/Preview/Db/PreviewMapper.php

@@ -29,6 +29,7 @@ class PreviewMapper extends QBMapper {
 	private const string TABLE_NAME = 'previews';
 	private const string LOCATION_TABLE_NAME = 'preview_locations';
 	private const string VERSION_TABLE_NAME = 'preview_versions';
+	private const int MAX_INSERT_PARAMETERS = 900;
 	public const MAX_CHUNK_SIZE = 1000;
 
 	// Columns selected by joinLocation() that do not belong to the previews table
@@ -59,6 +60,8 @@ public function insert(Entity $entity): Entity {
 		/** @var Preview $preview */
 		$preview = $entity;
 
+		// The version row reuses the preview id, so it has to exist before it is written.
+		$preview->generateId();
 		$preview->setMimetypeId($this->mimeTypeLoader->getId($preview->getMimeType()));
 		$preview->setSourceMimetypeId($this->mimeTypeLoader->getId($preview->getSourceMimeType()));
 
@@ -76,6 +79,77 @@ public function insert(Entity $entity): Entity {
 		return parent::insert($preview);
 	}
 
+	/**
+	 * Insert many migrated previews using multi-row INSERT statements.
+	 *
+	 * Only the columns set by the preview migration are written.
+	 *
+	 * @param list<Preview> $previews
+	 */
+	public function insertMany(array $previews): void {
+		$columns = [
+			'id' => IQueryBuilder::PARAM_STR,
+			'file_id' => IQueryBuilder::PARAM_INT,
+			'storage_id' => IQueryBuilder::PARAM_INT,
+			'old_file_id' => IQueryBuilder::PARAM_INT,
+			'location_id' => IQueryBuilder::PARAM_STR,
+			'width' => IQueryBuilder::PARAM_INT,
+			'height' => IQueryBuilder::PARAM_INT,
+			'mimetype_id' => IQueryBuilder::PARAM_INT,
+			'source_mimetype_id' => IQueryBuilder::PARAM_INT,
+			'mtime' => IQueryBuilder::PARAM_INT,
+			'size' => IQueryBuilder::PARAM_INT,
+			'max' => IQueryBuilder::PARAM_BOOL,
+			'cropped' => IQueryBuilder::PARAM_BOOL,
+			'encrypted' => IQueryBuilder::PARAM_BOOL,
+			'etag' => IQueryBuilder::PARAM_STR,
+		];
+
+		// Oracle before 23ai does not support multi-row VALUES.
+		$multiRow = $this->db->getDatabaseProvider() !== IDBConnection::PLATFORM_ORACLE;
+		$bulk = [];
+		foreach ($previews as $preview) {
+			if (!$multiRow || ($preview->getVersion() !== null && $preview->getVersion() !== '')) {
+				$this->insert($preview);
+				continue;
+			}
+			$preview->generateId();
+			$bulk[] = $preview;
+		}
+
+		$platform = $this->db->getDatabasePlatform();
+		$sql = 'INSERT INTO ' . $this->db->getQueryBuilder()->getTableName(self::TABLE_NAME)
+			. ' (' . implode(', ', array_map($platform->quoteSingleIdentifier(...), array_keys($columns))) . ') VALUES ';
+		$placeholders = '(' . implode(', ', array_fill(0, count($columns), '?')) . ')';
+		$types = array_values($columns);
+
+		foreach (array_chunk($bulk, intdiv(self::MAX_INSERT_PARAMETERS, count($columns))) as $chunk) {
+			$params = [];
+			$paramTypes = [];
+			foreach ($chunk as $preview) {
+				array_push($params,
+					$preview->getId(),
+					$preview->getFileId(),
+					$preview->getStorageId(),
+					$preview->getOldFileId(),
+					$preview->getLocationId(),
+					$preview->getWidth(),
+					$preview->getHeight(),
+					$this->mimeTypeLoader->getId($preview->getMimeType()),
+					$this->mimeTypeLoader->getId($preview->getSourceMimeType()),
+					$preview->getMtime(),
+					$preview->getSize(),
+					$preview->isMax(),
+					$preview->isCropped(),
+					$preview->isEncrypted(),
+					$preview->getEtag(),
+				);
+				array_push($paramTypes, ...$types);
+			}
+			$this->db->executeStatement($sql . implode(', ', array_fill(0, count($chunk), $placeholders)), $params, $paramTypes);
+		}
+	}
+
 	#[Override]
 	public function update(Entity $entity): Entity {
 		/** @var Preview $preview */