Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/playwright.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ jobs:
strategy:
fail-fast: false
matrix:
shardIndex: [1, 2, 3]
shardTotal: [3]
shardIndex: [1, 2, 3, 4]
shardTotal: [4]
steps:
- name: Checkout app
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
Expand Down
4 changes: 4 additions & 0 deletions lib/Cron/Cleanup.php
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ protected function run($argument): void {
$removedSteps = $this->sessionService->removeOrphanedSteps();
$this->logger->debug('Removed ' . $removedSteps . ' orphaned steps');

$this->logger->debug('Run cleanup job for saved steps');
$removedSavedSteps = $this->documentService->removeSavedSteps();
$this->logger->debug('Removed ' . $removedSavedSteps . ' saved steps');

$this->logger->debug('Run cleanup job for obsolete documents folders');
$this->documentService->cleanupOldDocumentsFolders();
}
Expand Down
32 changes: 30 additions & 2 deletions lib/Db/StepMapper.php
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,40 @@ public function clearAll(): void {
->executeStatement();
}

// not in use right now
public function deleteBeforeVersion(string $documentId, int $version): int {
/**
* Find documents with more than $keep steps that are older than $olderThan
* and included in the last saved version.
*
* @return list<array{documentId: string, lastSavedVersion: int}>
*/
public function findDocumentsWithSavedSteps(int $olderThan, int $keep): array {
$qb = $this->db->getQueryBuilder();
$result = $qb->select('st.document_id', 'd.last_saved_version')
->from($this->getTableName(), 'st')
->innerJoin('st', 'text_documents', 'd', $qb->expr()->eq('st.document_id', 'd.id'))
->where($qb->expr()->lt('st.id', 'd.last_saved_version'))
->andWhere($qb->expr()->lt('st.timestamp', $qb->createNamedParameter($olderThan, IQueryBuilder::PARAM_INT)))
->groupBy('st.document_id', 'd.last_saved_version')
->having($qb->expr()->gt($qb->func()->count('st.id'), $qb->createNamedParameter($keep, IQueryBuilder::PARAM_INT)))
->executeQuery();

$documents = [];
while ($row = $result->fetchAssociative()) {
$documents[] = [
'documentId' => (string)$row['document_id'],
'lastSavedVersion' => (int)$row['last_saved_version'],
];
}
$result->closeCursor();
return $documents;
}

public function deleteBeforeVersion(string $documentId, int $version, int $olderThan): int {
$qb = $this->db->getQueryBuilder();
return $qb->delete($this->getTableName())
->where($qb->expr()->eq('document_id', $qb->createNamedParameter($documentId)))
->andWhere($qb->expr()->lte('id', $qb->createNamedParameter($version)))
->andWhere($qb->expr()->lt('timestamp', $qb->createNamedParameter($olderThan, IQueryBuilder::PARAM_INT)))
->executeStatement();
}

Expand Down
29 changes: 28 additions & 1 deletion lib/Service/DocumentService.php
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ class DocumentService {
*/
public const int AUTOSAVE_MINIMUM_DELAY = 10;

/**
* Steps before the last saved version that recovering clients receive
* along with the document state, see #7692
*/
public const int STEPS_BEFORE_SAVED_VERSION = 200;

private bool $saveFromText = false;
private readonly ICache $cache;

Expand Down Expand Up @@ -275,7 +281,7 @@ public function addStep(Document $document, Session $session, array $steps, int
$this->logger->debug('Existing document, state file loaded ' . $documentId);
// If there were any queries in the steps, send all steps starting 200 steps before last save.
// Adding 200 previous steps to workaround race conditions where state with missing step got persisted in the document state. See #7692
$getStepsSinceVersion = $this->stepMapper->getBeforeVersion($documentId, $document->getLastSavedVersion(), 200);
$getStepsSinceVersion = $this->stepMapper->getBeforeVersion($documentId, $document->getLastSavedVersion(), self::STEPS_BEFORE_SAVED_VERSION);
} catch (NotFoundException) {
$this->logger->debug('Existing document, but no state file found for ' . $documentId);
// If there is no state file, include all the steps.
Expand Down Expand Up @@ -509,6 +515,27 @@ public function markAttachmentCleanup(Document $document): void {
$this->documentMapper->update($document);
}

/**
* Remove old steps that are included in the saved document state.
*
* Clients that still miss them recover from the document state, see addStep().
*/
public function removeSavedSteps(int $ageInSeconds = 604800): int {
$olderThan = time() - $ageInSeconds;
$removed = 0;
foreach ($this->stepMapper->findDocumentsWithSavedSteps($olderThan, self::STEPS_BEFORE_SAVED_VERSION) as $document) {
try {
$this->getStateFile($document['documentId']);
} catch (NotFoundException) {
// Without a document state recovering clients need all steps
continue;
}
$version = $this->stepMapper->getBeforeVersion($document['documentId'], $document['lastSavedVersion'], self::STEPS_BEFORE_SAVED_VERSION);
$removed += $this->stepMapper->deleteBeforeVersion($document['documentId'], $version, $olderThan);
}
return $removed;
}

public function hasUnsavedChanges(Document $document): bool {
if ($document->id === null) {
throw new RuntimeException('Document needs an id to check unsaved changes');
Expand Down
90 changes: 90 additions & 0 deletions playwright/e2e/synchronize-content/missing-steps.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/**
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

import type { Page, Request } from '@playwright/test'

import { expect, mergeTests } from '@playwright/test'
import { test as editorTest } from '../../support/fixtures/editor.ts'
import { Node } from '../../support/fixtures/Node.ts'
import { newUserPage } from '../../support/fixtures/random-user.ts'
import { test as uploadFileTest } from '../../support/fixtures/upload-file.ts'
import { EditorSection } from '../../support/sections/EditorSection.ts'

const test = mergeTests(editorTest, uploadFileTest)

const SESSION_REQUESTS = /\/apps\/text\/session\/\d+\/(sync|push)$/

/**
* Whether a request asks the server for the document state to recover from missing steps.
*
* @param request sent by the editor
*/
function isRecovery(request: Request): boolean {
return Boolean(request.postDataJSON()?.recoveryAttempt)
}

/**
* Hold back the sync and push requests of a page, like for a laptop that is suspended.
* Once resumed, leave out the older half of the steps it missed
* like a server that removed them in the meantime.
*
* @param page to suspend
* @return function to resume the page and the number of steps left out so far
*/
async function suspendAndDropSteps(page: Page) {
let resume = () => {}
const resumed = new Promise<void>((resolve) => {
resume = resolve
})
let dropUpTo: number | undefined
let dropped = 0
await page.route(SESSION_REQUESTS, async (route) => {
await resumed
const response = await route.fetch()
const json = await response.json().catch(() => undefined)
const steps: { version: number }[] | undefined = json?.steps
if (!steps?.length || isRecovery(route.request())) {
await route.fulfill({ response })
return
}
dropUpTo ??= steps[Math.floor(steps.length / 2) - 1]?.version
const kept = steps.filter((step) => dropUpTo === undefined || step.version > dropUpTo)
dropped += steps.length - kept.length
await route.fulfill({ response, json: { ...json, steps: kept } })
})
return { resume, dropped: () => dropped }
}

test.beforeEach(async ({ open }) => {
await open()
})

test('recovers missing steps from the document state', async ({
account,
baseURL,
browser,
editor,
file,
}) => {
await expect(editor.el).toBeVisible()
const otherPage = await newUserPage(browser, baseURL, account)
const other = new EditorSection(otherPage)
await new Node({ id: file.id, name: file.name, page: otherPage }).open()
await expect(other.el).toBeVisible()

const { resume, dropped } = await suspendAndDropSteps(otherPage)
// Pushes are sent at most every 200ms, so every key press is a separate step.
await editor.content.pressSequentially('abcdefghij', { delay: 300 })

const recovery = otherPage.waitForRequest((request) => SESSION_REQUESTS.test(request.url()) && isRecovery(request))
resume()
await recovery
expect(dropped()).toBeGreaterThan(0)
await expect(other.content).toContainText('abcdefghij')

await other.type('Back. ')
await expect(editor.content).toContainText('Back.')
await otherPage.close()
})
38 changes: 25 additions & 13 deletions playwright/support/fixtures/random-user.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/

import type { User as Account } from '@nextcloud/e2e-test-server'
import type { Browser, Page } from '@playwright/test'

import { createRandomUser, login } from '@nextcloud/e2e-test-server/playwright'
import { test as base } from '@playwright/test'
Expand All @@ -14,6 +15,29 @@ export interface UserFixture {
user: User
}

/**
* Open a new page that is logged in as the given account.
*
* @param browser to open the page in
* @param baseURL of the Nextcloud instance
* @param account to log in with
*/
export async function newUserPage(browser: Browser, baseURL: string | undefined, account: Account): Promise<Page> {
// Important: make sure we authenticate in a clean environment by unsetting storage state.
const page = await browser.newPage({
storageState: undefined,
baseURL,
})

await login(page.request, account)
const tokenResponse = await page.request.get('./csrftoken', {
failOnStatusCode: true,
})
const { token } = (await tokenResponse.json()) as { token: string }
page.context().setExtraHTTPHeaders({ requesttoken: token })
return page
}

/**
* This test fixture ensures a new random user is created and used for the test (current page)
*/
Expand All @@ -24,19 +48,7 @@ export const test = base.extend<UserFixture>({
await use(account)
},
page: async ({ account, browser, baseURL }, use) => {
// Important: make sure we authenticate in a clean environment by unsetting storage state.
const page = await browser.newPage({
storageState: undefined,
baseURL,
})

await login(page.request, account)
const tokenResponse = await page.request.get('./csrftoken', {
failOnStatusCode: true,
})
const { token } = (await tokenResponse.json()) as { token: string }
page.context().setExtraHTTPHeaders({ requesttoken: token })

const page = await newUserPage(browser, baseURL, account)
await use(page)
await page.close()
},
Expand Down
2 changes: 2 additions & 0 deletions tests/unit/CleanupTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ public function testMarksCleanupOnlyAfterAttachmentsAreCleaned(): void {
$this->sessionService->expects(self::once())->method('removeInactiveSessionsWithoutSteps');
$this->sessionService->expects(self::once())->method('removeOldSessions');
$this->sessionService->expects(self::once())->method('removeOrphanedSteps');
$this->documentService->expects(self::once())->method('removeSavedSteps');
$this->documentService->expects(self::once())->method('cleanupOldDocumentsFolders');

$this->cleanup->run([]);
Expand Down Expand Up @@ -129,6 +130,7 @@ public function testNotPermittedCleanupIsMarkedAndJobContinues(): void {
$this->sessionService->expects(self::once())->method('removeInactiveSessionsWithoutSteps');
$this->sessionService->expects(self::once())->method('removeOldSessions');
$this->sessionService->expects(self::once())->method('removeOrphanedSteps');
$this->documentService->expects(self::once())->method('removeSavedSteps');
$this->documentService->expects(self::once())->method('cleanupOldDocumentsFolders');

$this->cleanup->run([]);
Expand Down
Loading
Loading