Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions core/AppInfo/ConfigLexicon.php
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class ConfigLexicon implements ILexicon {
public const LASTCRON_TIMESTAMP = 'lastcron';

public const ON_DEMAND_PREVIEW_MIGRATION = 'on_demand_preview_migration';
public const PREVIEW_MIGRATION_MAX_PARALLEL_JOBS = 'preview_migration_max_parallel_jobs';

public const APPSTORE_LINK_SHOWN = 'appstore_link_shown';

Expand Down Expand Up @@ -106,6 +107,12 @@ public function getAppConfigs(): array {
defaultRaw: true,
definition: 'Whether on demand preview migration is enabled.'
),
new Entry(
key: self::PREVIEW_MIGRATION_MAX_PARALLEL_JOBS,
type: ValueType::INT,
defaultRaw: 2,
definition: 'Maximum number of preview migration background jobs running in parallel, -1 for no limit and 0 to disable the migration.',
),
new Entry(
key: self::APPSTORE_LINK_SHOWN,
type: ValueType::BOOL,
Expand Down
21 changes: 19 additions & 2 deletions core/BackgroundJobs/PreviewMigrationJob.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@

namespace OC\Core\BackgroundJobs;

use OC\BackgroundJob\JobList;
use OC\Core\AppInfo\ConfigLexicon;
use OC\Preview\Db\Preview;
use OC\Preview\PreviewMigrationService;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\BackgroundJob\TimedJob;
use OCP\Files\IRootFolder;
use OCP\IAppConfig;
Expand All @@ -31,7 +32,7 @@ public function __construct(
IConfig $config,
private readonly IRootFolder $rootFolder,
private readonly PreviewMigrationService $migrationService,
private readonly IJobList $jobList,
private readonly JobList $jobList,
private readonly LoggerInterface $logger,
) {
parent::__construct($time);
Expand All @@ -53,6 +54,22 @@ protected function run(mixed $argument): void {
return;
}

$maxParallelJobs = $this->appConfig->getValueInt('core', ConfigLexicon::PREVIEW_MIGRATION_MAX_PARALLEL_JOBS);
if ($maxParallelJobs === 0) {
$this->logger->debug('Preview migration of partition {partition} postponed: background migration is disabled.', [
'partition' => $partition,
]);
return;
}

if ($maxParallelJobs > 0 && $this->jobList->countReservedJobs(self::class, (int)$this->getId()) >= $maxParallelJobs) {
$this->logger->debug('Preview migration of partition {partition} postponed: {max} jobs are already running.', [
'partition' => $partition,
'max' => $maxParallelJobs,
]);
return;
}

if (!$this->runPartition($partition)) {
return;
}
Expand Down
28 changes: 27 additions & 1 deletion lib/private/BackgroundJob/JobList.php
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@

class JobList implements IJobList {
public const MAX_ARGUMENT_JSON_LENGTH = 32000;
/** Jobs reserved within this window count as running for hasReservedJob() and countReservedJobs() */
private const RUNNING_RESERVATION_WINDOW = 6 * 3600;

/** @var array<string, string> */
protected array $alreadyVisitedParallelBlocked = [];
Expand Down Expand Up @@ -402,7 +404,7 @@ public function hasReservedJob(?string $className = null): bool {
$query = $this->connection->getQueryBuilder();
$query->select('*')
->from('jobs')
->where($query->expr()->gt('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - 6 * 3600, IQueryBuilder::PARAM_INT)))
->where($query->expr()->gt('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - self::RUNNING_RESERVATION_WINDOW, IQueryBuilder::PARAM_INT)))
->setMaxResults(1);

if ($className !== null) {
Expand All @@ -420,6 +422,30 @@ public function hasReservedJob(?string $className = null): bool {
}
}

/**
* Counts the jobs of the passed class that were reserved to run
* in the last 6h
*
* @param class-string<IJob> $className
* @param ?int $excludeId job to leave out, e.g. the one currently running
*/
public function countReservedJobs(string $className, ?int $excludeId = null): int {
$query = $this->connection->getQueryBuilder();
$query->select($query->func()->count('*'))
->from('jobs')
->where($query->expr()->gt('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - self::RUNNING_RESERVATION_WINDOW, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('class', $query->createNamedParameter($className)));

if ($excludeId !== null) {
$query->andWhere($query->expr()->neq('id', $query->createNamedParameter($excludeId, IQueryBuilder::PARAM_INT)));
}

$result = $query->executeQuery();
$count = (int)$result->fetchOne();
$result->closeCursor();
return $count;
}

#[Override]
public function countByClass(): array {
$query = $this->connection->getQueryBuilder();
Expand Down
24 changes: 24 additions & 0 deletions tests/lib/BackgroundJob/JobListTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,30 @@ public function testHasReservedJobs(): void {
$this->assertTrue($this->instance->hasReservedJob(TestJob::class));
}

public function testCountReservedJobs(): void {
$this->clearJobsList();

$this->timeFactory->expects($this->atLeastOnce())
->method('getTime')
->willReturn(123456789);

$this->instance->add(new TestJob($this->timeFactory, $this, function (): void {
}), 1);
$this->instance->add(new TestJob($this->timeFactory, $this, function (): void {
}), 2);

$this->assertSame(0, $this->instance->countReservedJobs(TestJob::class));

$first = $this->instance->getNext();
$this->assertNotNull($first);
$this->assertSame(1, $this->instance->countReservedJobs(TestJob::class));
$this->assertSame(0, $this->instance->countReservedJobs(TestJob::class, (int)$first->getId()));
$this->assertSame(0, $this->instance->countReservedJobs(TestParallelAwareJob::class));

$this->assertNotNull($this->instance->getNext());
$this->assertSame(2, $this->instance->countReservedJobs(TestJob::class));
}

public function testHasReservedJobsAndParallelAwareJob(): void {
$this->clearJobsList();

Expand Down
55 changes: 55 additions & 0 deletions tests/lib/Preview/PreviewMigrationJobTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

namespace lib\Preview;

use OC\Core\AppInfo\ConfigLexicon;
use OC\Core\BackgroundJobs\PreviewMigrationJob;
use OC\Preview\Db\Preview;
use OC\Preview\Db\PreviewMapper;
Expand All @@ -17,6 +18,7 @@
use OC\Preview\Storage\StorageFactory;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\AppData\IAppDataFactory;
use OCP\Files\FileInfo;
use OCP\Files\IAppData;
Expand Down Expand Up @@ -44,6 +46,7 @@ class PreviewMigrationJobTest extends TestCase {
private IMimeTypeLoader&MockObject $mimeTypeLoader;
private IMimeTypeDetector&MockObject $mimeTypeDetector;
private LoggerInterface&MockObject $logger;
private int $maxParallelJobs = -1;

#[\Override]
public function setUp(): void {
Expand All @@ -58,6 +61,11 @@ public function setUp(): void {
$this->appConfig->expects($this->any())
->method('setValueBool')
->willReturn(true);
$this->appConfig->expects($this->any())
->method('getValueInt')
->willReturnCallback(fn (string $app, string $key): int => match ([$app, $key]) {
['core', ConfigLexicon::PREVIEW_MIGRATION_MAX_PARALLEL_JOBS] => $this->maxParallelJobs,
});
$this->storageFactory = Server::get(StorageFactory::class);
$this->previewService = Server::get(PreviewService::class);
$this->db = Server::get(IDBConnection::class);
Expand Down Expand Up @@ -287,6 +295,53 @@ public function testMigrationOnlyMigratesOwnPartition(): void {
$this->assertEquals(1, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5))));
}

#[TestDox('A partition must not be migrated while the maximum number of parallel jobs are already running')]
public function testMigrationRespectsMaxParallelJobs(): void {
$this->maxParallelJobs = 2;

$folder = $this->previewAppData->newFolder('5');
$folder->newFile('64-64-crop.jpg', 'abcdefg');

$firstRunningJobId = $this->insertReservedJob();
$this->insertReservedJob();

try {
$job = $this->createJob();
$this->invokePrivate($job, 'run', [['partition' => 5]]);
$this->assertEquals(1, count($this->previewAppData->getDirectoryListing()));
$this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5))));

Server::get(IJobList::class)->removeById($firstRunningJobId);

$this->invokePrivate($job, 'run', [['partition' => 5]]);
$this->assertEquals(0, count($this->previewAppData->getDirectoryListing()));
$this->assertEquals(1, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5))));
} finally {
Server::get(IJobList::class)->remove(PreviewMigrationJob::class);
}
}

private function insertReservedJob(): string {
$argument = ['partition' => $this->getUniqueID()];
Server::get(IJobList::class)->add(PreviewMigrationJob::class, $argument);

$argumentHash = hash('sha256', json_encode($argument));
$qb = $this->db->getQueryBuilder();
$qb->update('jobs')
->set('reserved_at', $qb->createNamedParameter(time(), IQueryBuilder::PARAM_INT))
->where($qb->expr()->eq('argument_hash', $qb->createNamedParameter($argumentHash)))
->executeStatement();

$qb = $this->db->getQueryBuilder();
$result = $qb->select('id')
->from('jobs')
->where($qb->expr()->eq('argument_hash', $qb->createNamedParameter($argumentHash)))
->executeQuery();
$id = (string)$result->fetchOne();
$result->closeCursor();
return $id;
}

#[TestDox('run() must complete without error when there is nothing to migrate')]
public function testMigrationWithoutAnyPreviews(): void {
$this->assertEquals(0, count($this->previewAppData->getDirectoryListing()));
Expand Down
Loading