From 65a90e7999c87f57f8efc294efe53f1213a04769 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Tue, 27 Oct 2026 17:41:08 +0100 Subject: [PATCH 1/3] feat: Allow to limit the amount of preview migration job running Signed-off-by: Carl Schwan --- core/AppInfo/ConfigLexicon.php | 1 + core/BackgroundJobs/PreviewMigrationJob.php | 30 +++++++++++ tests/lib/Preview/PreviewMigrationJobTest.php | 52 +++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/core/AppInfo/ConfigLexicon.php b/core/AppInfo/ConfigLexicon.php index ce8dd2b9c117b..ac7e05052d0d5 100644 --- a/core/AppInfo/ConfigLexicon.php +++ b/core/AppInfo/ConfigLexicon.php @@ -39,6 +39,7 @@ class ConfigLexicon implements ILexicon { public const LASTCRON_TIMESTAMP = 'lastcron'; public const ON_DEMAND_PREVIEW_MIGRATION = 'on_demand_preview_migration'; + public const PREVIEW_MIGRATION_MAX_PARALLEL_JOBS = 'preview_migration_max_parallel_jobs'; public function getStrictness(): Strictness { return Strictness::IGNORE; diff --git a/core/BackgroundJobs/PreviewMigrationJob.php b/core/BackgroundJobs/PreviewMigrationJob.php index fa3dff497d095..11381b350cd06 100644 --- a/core/BackgroundJobs/PreviewMigrationJob.php +++ b/core/BackgroundJobs/PreviewMigrationJob.php @@ -9,13 +9,16 @@ namespace OC\Core\BackgroundJobs; +use OC\Core\AppInfo\ConfigLexicon; use OC\Preview\PreviewMigrationService; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\IJobList; use OCP\BackgroundJob\TimedJob; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\IRootFolder; use OCP\IAppConfig; use OCP\IConfig; +use OCP\IDBConnection; use Override; use Psr\Log\LoggerInterface; @@ -32,6 +35,7 @@ public function __construct( private readonly PreviewMigrationService $migrationService, private readonly IJobList $jobList, private readonly LoggerInterface $logger, + private readonly IDBConnection $connection, ) { parent::__construct($time); $this->setTimeSensitivity(self::TIME_INSENSITIVE); @@ -52,6 +56,15 @@ protected function run(mixed $argument): void { return; } + $maxParallelJobs = $this->appConfig->getValueInt('core', ConfigLexicon::PREVIEW_MIGRATION_MAX_PARALLEL_JOBS); + if ($maxParallelJobs > 0 && $this->countOtherRunningJobs() >= $maxParallelJobs) { + $this->logger->debug('Preview migration of partition {partition} postponed: {max} jobs are already running.', [ + 'partition' => $partition, + 'max' => $maxParallelJobs, + ]); + return; + } + if (!$this->runPartition($partition)) { return; } @@ -130,6 +143,23 @@ private function runPartition(int $partition): bool { return true; } + /** + * Count the other preview migration jobs currently reserved by a cron worker, + * using the same staleness window as {@see \OCP\BackgroundJob\IJobList::hasReservedJob()}. + */ + private function countOtherRunningJobs(): int { + $qb = $this->connection->getQueryBuilder(); + $qb->select($qb->func()->count('*')) + ->from('jobs') + ->where($qb->expr()->eq('class', $qb->createNamedParameter(self::class))) + ->andWhere($qb->expr()->gt('reserved_at', $qb->createNamedParameter($this->time->getTime() - 6 * 3600, IQueryBuilder::PARAM_INT))) + ->andWhere($qb->expr()->neq('id', $qb->createNamedParameter((int)$this->getId(), IQueryBuilder::PARAM_INT))); + $result = $qb->executeQuery(); + $count = (int)$result->fetchOne(); + $result->closeCursor(); + return $count; + } + private function belongsToPartition(string $folderName, int $partition): bool { if (ctype_digit($folderName)) { return ((int)$folderName % self::PARTITIONS) === $partition; diff --git a/tests/lib/Preview/PreviewMigrationJobTest.php b/tests/lib/Preview/PreviewMigrationJobTest.php index 9cf8b2dd1460b..53e5e8542fe31 100644 --- a/tests/lib/Preview/PreviewMigrationJobTest.php +++ b/tests/lib/Preview/PreviewMigrationJobTest.php @@ -9,6 +9,7 @@ namespace lib\Preview; +use OC\Core\AppInfo\ConfigLexicon; use OC\Core\BackgroundJobs\PreviewMigrationJob; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; @@ -17,6 +18,7 @@ use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\IJobList; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\FileInfo; use OCP\Files\IAppData; @@ -149,6 +151,7 @@ private function createJob(): PreviewMigrationJob { ), Server::get(IJobList::class), $this->logger, + $this->db, ); } @@ -287,6 +290,55 @@ public function testMigrationOnlyMigratesOwnPartition(): void { $this->assertEquals(1, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); } + #[TestDox('A partition must not be migrated while the maximum number of parallel jobs are already running')] + public function testMigrationRespectsMaxParallelJobs(): void { + $this->appConfig->method('getValueInt') + ->with('core', ConfigLexicon::PREVIEW_MIGRATION_MAX_PARALLEL_JOBS) + ->willReturn(2); + + $folder = $this->previewAppData->newFolder('5'); + $folder->newFile('64-64-crop.jpg', 'abcdefg'); + + $firstRunningJobId = $this->insertReservedJob(); + $this->insertReservedJob(); + + try { + $job = $this->createJob(); + $this->invokePrivate($job, 'run', [['partition' => 5]]); + $this->assertEquals(1, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); + + Server::get(IJobList::class)->removeById($firstRunningJobId); + + $this->invokePrivate($job, 'run', [['partition' => 5]]); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(1, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); + } finally { + Server::get(IJobList::class)->remove(PreviewMigrationJob::class); + } + } + + private function insertReservedJob(): string { + $argument = ['partition' => $this->getUniqueID()]; + Server::get(IJobList::class)->add(PreviewMigrationJob::class, $argument); + + $argumentHash = hash('sha256', json_encode($argument)); + $qb = $this->db->getQueryBuilder(); + $qb->update('jobs') + ->set('reserved_at', $qb->createNamedParameter(time(), IQueryBuilder::PARAM_INT)) + ->where($qb->expr()->eq('argument_hash', $qb->createNamedParameter($argumentHash))) + ->executeStatement(); + + $qb = $this->db->getQueryBuilder(); + $result = $qb->select('id') + ->from('jobs') + ->where($qb->expr()->eq('argument_hash', $qb->createNamedParameter($argumentHash))) + ->executeQuery(); + $id = (string)$result->fetchOne(); + $result->closeCursor(); + return $id; + } + #[TestDox('run() must complete without error when there is nothing to migrate')] public function testMigrationWithoutAnyPreviews(): void { $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); From d73ba4ab79880306e0bbb4d07d95842f3339130c Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Fri, 6 Nov 2026 16:07:22 +0100 Subject: [PATCH 2/3] feat(preview): Make it possible to completely stop the preview migration job Signed-off-by: Carl Schwan --- core/BackgroundJobs/PreviewMigrationJob.php | 7 +++++++ tests/lib/Preview/PreviewMigrationJobTest.php | 10 +++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/core/BackgroundJobs/PreviewMigrationJob.php b/core/BackgroundJobs/PreviewMigrationJob.php index 11381b350cd06..fd768c11d55b1 100644 --- a/core/BackgroundJobs/PreviewMigrationJob.php +++ b/core/BackgroundJobs/PreviewMigrationJob.php @@ -57,6 +57,13 @@ protected function run(mixed $argument): void { } $maxParallelJobs = $this->appConfig->getValueInt('core', ConfigLexicon::PREVIEW_MIGRATION_MAX_PARALLEL_JOBS); + if ($maxParallelJobs === 0) { + $this->logger->debug('Preview migration of partition {partition} postponed: background migration is disabled.', [ + 'partition' => $partition, + ]); + return; + } + if ($maxParallelJobs > 0 && $this->countOtherRunningJobs() >= $maxParallelJobs) { $this->logger->debug('Preview migration of partition {partition} postponed: {max} jobs are already running.', [ 'partition' => $partition, diff --git a/tests/lib/Preview/PreviewMigrationJobTest.php b/tests/lib/Preview/PreviewMigrationJobTest.php index 53e5e8542fe31..eece7097d02e8 100644 --- a/tests/lib/Preview/PreviewMigrationJobTest.php +++ b/tests/lib/Preview/PreviewMigrationJobTest.php @@ -46,6 +46,7 @@ class PreviewMigrationJobTest extends TestCase { private IMimeTypeLoader&MockObject $mimeTypeLoader; private IMimeTypeDetector&MockObject $mimeTypeDetector; private LoggerInterface&MockObject $logger; + private int $maxParallelJobs = -1; #[\Override] public function setUp(): void { @@ -60,6 +61,11 @@ public function setUp(): void { $this->appConfig->expects($this->any()) ->method('setValueBool') ->willReturn(true); + $this->appConfig->expects($this->any()) + ->method('getValueInt') + ->willReturnCallback(fn (string $app, string $key): int => match ([$app, $key]) { + ['core', ConfigLexicon::PREVIEW_MIGRATION_MAX_PARALLEL_JOBS] => $this->maxParallelJobs, + }); $this->storageFactory = Server::get(StorageFactory::class); $this->previewService = Server::get(PreviewService::class); $this->db = Server::get(IDBConnection::class); @@ -292,9 +298,7 @@ public function testMigrationOnlyMigratesOwnPartition(): void { #[TestDox('A partition must not be migrated while the maximum number of parallel jobs are already running')] public function testMigrationRespectsMaxParallelJobs(): void { - $this->appConfig->method('getValueInt') - ->with('core', ConfigLexicon::PREVIEW_MIGRATION_MAX_PARALLEL_JOBS) - ->willReturn(2); + $this->maxParallelJobs = 2; $folder = $this->previewAppData->newFolder('5'); $folder->newFile('64-64-crop.jpg', 'abcdefg'); From 198b3ca2a658f13f633bdd8e467b612b517a8245 Mon Sep 17 00:00:00 2001 From: Peter Ringelmann Date: Wed, 11 Nov 2026 11:29:11 +0100 Subject: [PATCH 3/3] refactor(core): move reserved job count to JobList. Signed-off-by: Peter Ringelmann Signed-off-by: Louis Chmn --- core/AppInfo/ConfigLexicon.php | 6 ++++ core/BackgroundJobs/PreviewMigrationJob.php | 26 ++--------------- lib/private/BackgroundJob/JobList.php | 28 ++++++++++++++++++- tests/lib/BackgroundJob/JobListTest.php | 24 ++++++++++++++++ tests/lib/Preview/PreviewMigrationJobTest.php | 1 - 5 files changed, 60 insertions(+), 25 deletions(-) diff --git a/core/AppInfo/ConfigLexicon.php b/core/AppInfo/ConfigLexicon.php index ac7e05052d0d5..1a05fe959abe4 100644 --- a/core/AppInfo/ConfigLexicon.php +++ b/core/AppInfo/ConfigLexicon.php @@ -101,6 +101,12 @@ public function getAppConfigs(): array { defaultRaw: true, definition: 'Whether on demand preview migration is enabled.' ), + new Entry( + key: self::PREVIEW_MIGRATION_MAX_PARALLEL_JOBS, + type: ValueType::INT, + defaultRaw: 2, + definition: 'Maximum number of preview migration background jobs running in parallel, -1 for no limit and 0 to disable the migration.', + ), new Entry( key: self::SHARE_REPAIR_REMOVED_TRAILING_SLASHES, type: ValueType::BOOL, diff --git a/core/BackgroundJobs/PreviewMigrationJob.php b/core/BackgroundJobs/PreviewMigrationJob.php index fd768c11d55b1..0ffdd19d77693 100644 --- a/core/BackgroundJobs/PreviewMigrationJob.php +++ b/core/BackgroundJobs/PreviewMigrationJob.php @@ -9,16 +9,14 @@ namespace OC\Core\BackgroundJobs; +use OC\BackgroundJob\JobList; use OC\Core\AppInfo\ConfigLexicon; use OC\Preview\PreviewMigrationService; use OCP\AppFramework\Utility\ITimeFactory; -use OCP\BackgroundJob\IJobList; use OCP\BackgroundJob\TimedJob; -use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\IRootFolder; use OCP\IAppConfig; use OCP\IConfig; -use OCP\IDBConnection; use Override; use Psr\Log\LoggerInterface; @@ -33,9 +31,8 @@ public function __construct( IConfig $config, private readonly IRootFolder $rootFolder, private readonly PreviewMigrationService $migrationService, - private readonly IJobList $jobList, + private readonly JobList $jobList, private readonly LoggerInterface $logger, - private readonly IDBConnection $connection, ) { parent::__construct($time); $this->setTimeSensitivity(self::TIME_INSENSITIVE); @@ -64,7 +61,7 @@ protected function run(mixed $argument): void { return; } - if ($maxParallelJobs > 0 && $this->countOtherRunningJobs() >= $maxParallelJobs) { + if ($maxParallelJobs > 0 && $this->jobList->countReservedJobs(self::class, (int)$this->getId()) >= $maxParallelJobs) { $this->logger->debug('Preview migration of partition {partition} postponed: {max} jobs are already running.', [ 'partition' => $partition, 'max' => $maxParallelJobs, @@ -150,23 +147,6 @@ private function runPartition(int $partition): bool { return true; } - /** - * Count the other preview migration jobs currently reserved by a cron worker, - * using the same staleness window as {@see \OCP\BackgroundJob\IJobList::hasReservedJob()}. - */ - private function countOtherRunningJobs(): int { - $qb = $this->connection->getQueryBuilder(); - $qb->select($qb->func()->count('*')) - ->from('jobs') - ->where($qb->expr()->eq('class', $qb->createNamedParameter(self::class))) - ->andWhere($qb->expr()->gt('reserved_at', $qb->createNamedParameter($this->time->getTime() - 6 * 3600, IQueryBuilder::PARAM_INT))) - ->andWhere($qb->expr()->neq('id', $qb->createNamedParameter((int)$this->getId(), IQueryBuilder::PARAM_INT))); - $result = $qb->executeQuery(); - $count = (int)$result->fetchOne(); - $result->closeCursor(); - return $count; - } - private function belongsToPartition(string $folderName, int $partition): bool { if (ctype_digit($folderName)) { return ((int)$folderName % self::PARTITIONS) === $partition; diff --git a/lib/private/BackgroundJob/JobList.php b/lib/private/BackgroundJob/JobList.php index dbd24767a90db..eebe6c4f0e407 100644 --- a/lib/private/BackgroundJob/JobList.php +++ b/lib/private/BackgroundJob/JobList.php @@ -27,6 +27,8 @@ class JobList implements IJobList { public const MAX_ARGUMENT_JSON_LENGTH = 32000; + /** Jobs reserved within this window count as running for hasReservedJob() and countReservedJobs() */ + private const RUNNING_RESERVATION_WINDOW = 6 * 3600; /** @var array */ protected array $alreadyVisitedParallelBlocked = []; @@ -398,7 +400,7 @@ public function hasReservedJob(?string $className = null): bool { $query = $this->connection->getQueryBuilder(); $query->select('*') ->from('jobs') - ->where($query->expr()->gt('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - 6 * 3600, IQueryBuilder::PARAM_INT))) + ->where($query->expr()->gt('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - self::RUNNING_RESERVATION_WINDOW, IQueryBuilder::PARAM_INT))) ->setMaxResults(1); if ($className !== null) { @@ -416,6 +418,30 @@ public function hasReservedJob(?string $className = null): bool { } } + /** + * Counts the jobs of the passed class that were reserved to run + * in the last 6h + * + * @param class-string $className + * @param ?int $excludeId job to leave out, e.g. the one currently running + */ + public function countReservedJobs(string $className, ?int $excludeId = null): int { + $query = $this->connection->getQueryBuilder(); + $query->select($query->func()->count('*')) + ->from('jobs') + ->where($query->expr()->gt('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - self::RUNNING_RESERVATION_WINDOW, IQueryBuilder::PARAM_INT))) + ->andWhere($query->expr()->eq('class', $query->createNamedParameter($className))); + + if ($excludeId !== null) { + $query->andWhere($query->expr()->neq('id', $query->createNamedParameter($excludeId, IQueryBuilder::PARAM_INT))); + } + + $result = $query->executeQuery(); + $count = (int)$result->fetchOne(); + $result->closeCursor(); + return $count; + } + #[Override] public function countByClass(): array { $query = $this->connection->getQueryBuilder(); diff --git a/tests/lib/BackgroundJob/JobListTest.php b/tests/lib/BackgroundJob/JobListTest.php index ff10b1edecde0..7ebb1e3dd8916 100644 --- a/tests/lib/BackgroundJob/JobListTest.php +++ b/tests/lib/BackgroundJob/JobListTest.php @@ -323,6 +323,30 @@ public function testHasReservedJobs(): void { $this->assertTrue($this->instance->hasReservedJob(TestJob::class)); } + public function testCountReservedJobs(): void { + $this->clearJobsList(); + + $this->timeFactory->expects($this->atLeastOnce()) + ->method('getTime') + ->willReturn(123456789); + + $this->instance->add(new TestJob($this->timeFactory, $this, function (): void { + }), 1); + $this->instance->add(new TestJob($this->timeFactory, $this, function (): void { + }), 2); + + $this->assertSame(0, $this->instance->countReservedJobs(TestJob::class)); + + $first = $this->instance->getNext(); + $this->assertNotNull($first); + $this->assertSame(1, $this->instance->countReservedJobs(TestJob::class)); + $this->assertSame(0, $this->instance->countReservedJobs(TestJob::class, (int)$first->getId())); + $this->assertSame(0, $this->instance->countReservedJobs(TestParallelAwareJob::class)); + + $this->assertNotNull($this->instance->getNext()); + $this->assertSame(2, $this->instance->countReservedJobs(TestJob::class)); + } + public function testHasReservedJobsAndParallelAwareJob(): void { $this->clearJobsList(); diff --git a/tests/lib/Preview/PreviewMigrationJobTest.php b/tests/lib/Preview/PreviewMigrationJobTest.php index eece7097d02e8..7fbb3d5528ecd 100644 --- a/tests/lib/Preview/PreviewMigrationJobTest.php +++ b/tests/lib/Preview/PreviewMigrationJobTest.php @@ -157,7 +157,6 @@ private function createJob(): PreviewMigrationJob { ), Server::get(IJobList::class), $this->logger, - $this->db, ); }