diff --git a/docs/debugging.md b/docs/debugging.md new file mode 100644 index 000000000..7c0ba25fc --- /dev/null +++ b/docs/debugging.md @@ -0,0 +1,6 @@ +## Debugging in browser + +To run app in browser mode: +- run `npm run serve --workspace app` +- go to http://localhost:8080/window-main.html +- open console and enable IDB storage via command `localStorage.storageType = 'idb'` \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 9653dee7a..be3cfc8f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4895,6 +4895,16 @@ "node": ">= 14.17.5" } }, + "node_modules/@electron-internal/extract-zip": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@electron-internal/extract-zip/-/extract-zip-1.0.5.tgz", + "integrity": "sha512-+bqFCP98pLI0Tt0XQo1TmlXtwjWchISndDOxCkEcIuUgXWpBnLyRI+2DU+mesvnMMX6L1XDqYNA0lXNDHd/yiA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=22.12.0" + } + }, "node_modules/@electron/asar": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/@electron/asar/-/asar-3.4.1.tgz", @@ -17407,22 +17417,22 @@ } }, "node_modules/electron": { - "version": "39.8.10", - "resolved": "https://registry.npmjs.org/electron/-/electron-39.8.10.tgz", - "integrity": "sha512-zbYtGPYUI7PzqLAzkk21Rk6j67WN0hxn0Mq/njErZo1d0HSf33is4f8ICI5fMLy5vYe0JtCtM5sYunNOaochSQ==", + "version": "44.4.5", + "resolved": "https://registry.npmjs.org/electron/-/electron-44.4.5.tgz", + "integrity": "sha512-SjgoaeYsSWZfJzubgQU7juvuXMTvn6/e1gAHdGFA/yuMbpF+I+skYqIJ6DdXBXHeWbkbpNpmwZCTpiivtlWZSw==", "dev": true, - "hasInstallScript": true, "license": "MIT", "dependencies": { - "@electron/get": "^2.0.0", - "@types/node": "^22.7.7", - "extract-zip": "^2.0.1" + "@electron-internal/extract-zip": "^1.0.1", + "@electron/get": "^5.0.0", + "@types/node": "^24.9.0" }, "bin": { - "electron": "cli.js" + "electron": "cli.js", + "install-electron": "install.js" }, "engines": { - "node": ">= 12.20.55" + "node": ">= 22.12.0" } }, "node_modules/electron-devtools-installer": { @@ -17788,89 +17798,56 @@ } }, "node_modules/electron/node_modules/@electron/get": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@electron/get/-/get-2.0.3.tgz", - "integrity": "sha512-Qkzpg2s9GnVV2I2BjRksUi43U5e6+zaQMcjoJy0C+C5oxaKl+fmckGDQFtRpZpZV0NQekuZZ+tGz7EA9TVnQtQ==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@electron/get/-/get-5.1.0.tgz", + "integrity": "sha512-3kSBtG8ObcTVfXanm5vVJ6UnBLEVmVsRk1M+vGqCuMBV+XLCbJYuWQful+yIy0GQDsSlK0kHEriEHn7SPk4EnA==", "dev": true, "license": "MIT", "dependencies": { "debug": "^4.1.1", - "env-paths": "^2.2.0", - "fs-extra": "^8.1.0", - "got": "^11.8.5", + "env-paths": "^3.0.0", + "graceful-fs": "^4.2.11", "progress": "^2.0.3", - "semver": "^6.2.0", + "semver": "^7.6.3", "sumchecker": "^3.0.1" }, "engines": { - "node": ">=12" + "node": ">=22.12.0" }, "optionalDependencies": { - "global-agent": "^3.0.0" + "undici": "^7.24.4" } }, "node_modules/electron/node_modules/@types/node": { - "version": "22.19.15", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.15.tgz", - "integrity": "sha512-F0R/h2+dsy5wJAUe3tAU6oqa2qbWY5TpNfL/RGmo1y38hiyO1w3x2jPtt76wmuaJI4DQnOBu21cNXQ2STIUUWg==", + "version": "24.19.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.19.0.tgz", + "integrity": "sha512-zY+5tKxXdhGh1PYI0ac+7juvEu4OI6vWtVVoj5i2m42jxAY1U+zHGt6QCyOFwykdP62sM3MJ9stoYYUw5aCWew==", "dev": true, "license": "MIT", "dependencies": { - "undici-types": "~6.21.0" + "undici-types": ">=7.24.0 <7.24.7" } }, - "node_modules/electron/node_modules/fs-extra": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", - "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "node_modules/electron/node_modules/env-paths": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", + "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", "dev": true, "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - }, "engines": { - "node": ">=6 <7 || >=8" - } - }, - "node_modules/electron/node_modules/jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", - "dev": true, - "license": "MIT", - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/electron/node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/electron/node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", "dev": true, "license": "MIT" }, - "node_modules/electron/node_modules/universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -33239,7 +33216,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "engines": { "node": ">=20.18.1" } @@ -35663,7 +35639,7 @@ "copy-webpack-plugin": "^13.0.1", "cross-env": "^10.1.0", "css-loader": "^7.1.3", - "electron": "^39.8.2", + "electron": "^44.4.5", "electron-devtools-installer": "^4.0.0", "fork-ts-checker-webpack-plugin": "^9.1.0", "html-webpack-plugin": "^5.6.6", diff --git a/packages/app/package.json b/packages/app/package.json index 3e28407db..3e1b30fa2 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -76,7 +76,7 @@ "copy-webpack-plugin": "^13.0.1", "cross-env": "^10.1.0", "css-loader": "^7.1.3", - "electron": "^39.8.2", + "electron": "^44.4.5", "electron-devtools-installer": "^4.0.0", "fork-ts-checker-webpack-plugin": "^9.1.0", "html-webpack-plugin": "^5.6.6", diff --git a/packages/app/src/__tests__/utils/fs.ts b/packages/app/src/__tests__/utils/fs.ts new file mode 100644 index 000000000..3469f489e --- /dev/null +++ b/packages/app/src/__tests__/utils/fs.ts @@ -0,0 +1,7 @@ +/* eslint-disable @typescript-eslint/no-base-to-string */ +import { vol } from 'memfs'; + +export const readdir = (path: string) => + vol + .readdirSync(path, { recursive: true }) + .sort((a, b) => String(a).localeCompare(String(b))); diff --git a/packages/app/src/core/database/ManagedDatabase.ts b/packages/app/src/core/database/ManagedDatabase.ts index 538453f58..bb5813aea 100644 --- a/packages/app/src/core/database/ManagedDatabase.ts +++ b/packages/app/src/core/database/ManagedDatabase.ts @@ -44,6 +44,15 @@ export type Options = { }; }; +const requestRunOnIdle = (cb: () => void, deadline: number) => { + if (typeof globalThis.requestIdleCallback === 'function') { + globalThis.requestIdleCallback(cb, { timeout: deadline }); + return; + } + + setTimeout(cb, 0); +}; + /** * Class to manage database data synchronization */ @@ -61,9 +70,14 @@ export class ManagedDatabase implements IManagedDatabase { this.dbFile = dbFile; // Auto sync changes - this.debouncedSync = debounce(this.sync, { - wait: sync.delay, - }); + this.debouncedSync = debounce( + () => { + requestRunOnIdle(() => this.sync(), sync.deadline); + }, + { + wait: sync.delay, + }, + ); this.cleanups.push(dbContainer.onChanged.watch(this.debouncedSync)); } diff --git a/packages/app/src/electron/requests/storage/ElectronFilesController.integration.test.ts b/packages/app/src/electron/requests/storage/ElectronFilesController.integration.test.ts index 7f9cc795d..0b4b01f04 100644 --- a/packages/app/src/electron/requests/storage/ElectronFilesController.integration.test.ts +++ b/packages/app/src/electron/requests/storage/ElectronFilesController.integration.test.ts @@ -1,118 +1,363 @@ import { vol } from 'memfs'; +import { getRandomBytes } from '@core/encryption/utils/random'; +import { ipcMainHandler } from '@electron/utils/ipc/ipcMainHandler'; +import { readdir } from '@tests/utils/fs'; -import { enableStorage } from './main'; +import { createStorageBackend, enableStorage } from './main'; import { ElectronFilesController, storageApi } from './renderer'; +import { storageChannel } from '.'; vi.mock('fs', () => vi.importActual('@mocks/fs')); +vi.mock('node:fs', () => vi.importActual('@mocks/fs')); vi.mock('fs/promises', () => vi.importActual('@mocks/fs/promises')); -vi.mock('recursive-readdir', () => vi.importActual('@mocks/recursive-readdir')); +vi.mock('node:fs/promises', () => vi.importActual('@mocks/fs/promises')); vi.mock('electron', () => vi.importActual('@mocks/electron')); -const storageHandlerCleanup = enableStorage(); -const filesController = new ElectronFilesController(storageApi, 'vaultDir'); - const getBufferFromText = (text: string) => new Uint8Array(Buffer.from(text)).buffer; -test('Write files', async () => { - expect(vol.readdirSync('/', { recursive: true })).toEqual([]); - - await filesController.write( - 'test.txt', - new Uint8Array(Buffer.from('File text content')).buffer, - ); - await filesController.write( - '/foo/bar/baz/test.txt', - new Uint8Array(Buffer.from('File text content')).buffer, +const createFiles = ( + files: ElectronFilesController, + fileNames: string[], + content?: ArrayBuffer, +) => + Promise.all( + fileNames.map((id) => files.write(id, content ?? new Uint8Array(100).buffer)), ); - expect(vol.readdirSync('/', { recursive: true })).toMatchSnapshot( - 'FS structure after writing', - ); - await expect(filesController.list()).resolves.toMatchSnapshot( - 'Controller files list after writing', +describe('Chunking', () => { + const cleanups = [] as (() => void)[]; + beforeAll(() => { + vol.reset(); + cleanups.push(enableStorage()); + }); + afterAll(() => { + cleanups.forEach((cleanup) => cleanup()); + }); + + test.each([ + { size: 99, chunkSize: 5 }, + { size: 20, chunkSize: 5 }, + { size: 5, chunkSize: 5 }, + { size: 4.9, chunkSize: 5 }, + { size: 0.1, chunkSize: 5 }, + ])( + 'File reading returns exact equal buffer as original data (chunk size $chunkSize Mb, file size $size Mb)', + async ({ size, chunkSize }) => { + const filesController = new ElectronFilesController(storageApi, 'size-test', { + chunkSize, + }); + const originalBuffer = getRandomBytes(1024 ** 2 * size); + + await filesController.write('data', originalBuffer.buffer); + + const bytesFromDisk = await filesController.get('data'); + expect(bytesFromDisk).toBeInstanceOf(ArrayBuffer); + expect(Buffer.from(originalBuffer).equals(Buffer.from(bytesFromDisk!))).toBe( + true, + ); + }, ); }); -test('Check files content', async () => { - await expect( - filesController.list().then((paths) => - Promise.all( - paths.map((path) => - filesController.get(path).then((buffer) => { - if (!buffer) throw new Error('Buffer not found'); - return { - path, - content: Buffer.from(buffer).toString(), - }; - }), +describe('Basic ops', () => { + const cleanups = [] as (() => void)[]; + beforeAll(() => { + cleanups.push(enableStorage()); + }); + afterAll(() => { + cleanups.forEach((cleanup) => cleanup()); + }); + beforeEach(() => { + vol.reset(); + }); + + test('File uploading persist data on disk', async () => { + // No files in FS + expect(readdir('/')).toEqual([]); + + const filesController = new ElectronFilesController(storageApi, 'vaultDir'); + + // Write files + const contentSample = getBufferFromText('File text content'); + await filesController.write('test.txt', contentSample); + await filesController.write('/foo/bar/baz/test.txt', contentSample); + + // Explore files list + await expect(filesController.list()).resolves.toMatchSnapshot( + 'Controller files list after writing', + ); + + expect(readdir('/'), 'Explore FS').toMatchSnapshot('FS structure after writing'); + + // Explore files content + await expect( + filesController.list().then((paths) => + Promise.all( + paths.map((path) => + filesController.get(path).then((buffer) => { + if (!buffer) throw new Error('Buffer not found'); + return { + path, + content: Buffer.from(buffer).toString(), + }; + }), + ), ), ), - ), - ).resolves.toMatchSnapshot('Files content'); -}); + ).resolves.toMatchSnapshot('Files content'); + }); -test('File may be accessed via absolute and relevant paths both', async () => { - await expect(filesController.get('test.txt')).resolves.toStrictEqual( - getBufferFromText('File text content'), - ); - await expect(filesController.get('/test.txt')).resolves.toStrictEqual( - getBufferFromText('File text content'), - ); - await expect(filesController.get('////test.txt')).resolves.toStrictEqual( - getBufferFromText('File text content'), - ); - await expect(filesController.get('/foo/../test.txt')).resolves.toStrictEqual( - getBufferFromText('File text content'), - ); + test('Fetching for directory returns nothing', async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + await createFiles(files, ['/foo/bar/baz/test.txt']); - await expect(filesController.get('/foo/bar/baz/test.txt')).resolves.toStrictEqual( - getBufferFromText('File text content'), - ); - await expect(filesController.get('./foo/bar/baz/test.txt')).resolves.toStrictEqual( - getBufferFromText('File text content'), - ); - await expect(filesController.get('foo/bar/baz/test.txt')).resolves.toStrictEqual( - getBufferFromText('File text content'), - ); + await expect(files.get('/foo')).resolves.toBe(null); + }); }); -describe('Deletion tests', () => { - const files = new ElectronFilesController(storageApi, 'deletionTests'); +describe('Files paths', () => { + const cleanups = [] as (() => void)[]; + beforeAll(() => { + cleanups.push(enableStorage()); + }); + afterAll(() => { + cleanups.forEach((cleanup) => cleanup()); + }); + beforeEach(() => { + vol.reset(); + }); + + const contentSample = getBufferFromText('File text content'); + + test('Files can be fetched by paths with no leading slash', async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + await createFiles(files, ['test.txt', '/foo/bar/baz/test.txt'], contentSample); + + await expect(files.get('test.txt')).resolves.toStrictEqual(contentSample); + await expect(files.get('foo/bar/baz/test.txt')).resolves.toStrictEqual( + contentSample, + ); + }); + + test('Files can be fetched by paths with leading slash', async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + await createFiles(files, ['test.txt', '/foo/bar/baz/test.txt'], contentSample); + + await expect(files.get('/test.txt')).resolves.toStrictEqual(contentSample); + await expect(files.get('/foo/bar/baz/test.txt')).resolves.toStrictEqual( + contentSample, + ); - test('Add files', async () => { - await expect( - files.write('/root.bytes', new Uint8Array().buffer), - ).resolves.not.toThrow(); - await expect( - files.write('/bytes1', new Uint8Array().buffer), - ).resolves.not.toThrow(); - await expect( - files.write('/bytes2', new Uint8Array().buffer), - ).resolves.not.toThrow(); - await expect( - files.write('/bytes3', new Uint8Array().buffer), - ).resolves.not.toThrow(); - await expect( - files.write('/foo/bytes1', new Uint8Array().buffer), - ).resolves.not.toThrow(); await expect( - files.write('/foo/bytes2', new Uint8Array().buffer), - ).resolves.not.toThrow(); + files.get('////test.txt'), + 'Leading slash can be repeated', + ).resolves.toStrictEqual(contentSample); + }); + + test('Files can be fetched by relative paths', async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + await createFiles(files, ['test.txt', '/foo/bar/baz/test.txt'], contentSample); + + await expect(files.get('/foo/../test.txt')).resolves.toStrictEqual(contentSample); + await expect(files.get('./foo/bar/baz/test.txt')).resolves.toStrictEqual( + contentSample, + ); + }); +}); + +describe('Path traversal isolation', () => { + const cleanups = [] as (() => void)[]; + beforeAll(() => { + cleanups.push(enableStorage()); + }); + afterAll(() => { + cleanups.forEach((cleanup) => cleanup()); + }); + + const contentSample = getBufferFromText('File text content'); + beforeEach(() => { + vol.reset(); + + vol.mkdirSync('/home/userData/appDir/app/vaultDir', { recursive: true }); + vol.writeFileSync( + '/home/userData/appDir/app/vaultDir/secret.txt', + Buffer.from(new Uint8Array(contentSample)), + ); + }); + + test("Client can't traverse out of root directory", async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + + await expect(files.get('../vault2/test.bytes')).rejects.toThrowError( + 'Resolved path is out of root directory', + ); + + await expect(files.get('../secret.txt')).rejects.toThrowError( + 'Resolved path is out of root directory', + ); + }); + + test('Client cannot read from a sibling directory with a matching root prefix', async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + vol.mkdirSync('/home/userData/appDir/app/vaultDir2', { recursive: true }); + vol.writeFileSync( + '/home/userData/appDir/app/vaultDir2/secret.txt', + 'sibling secret', + ); + + await expect(files.get('../vaultDir2/secret.txt')).rejects.toThrow( + 'Resolved path is out of root directory', + ); + }); + + test('Client cannot overwrite a file in a sibling directory with a matching root prefix', async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + const siblingSecretPath = '/home/userData/appDir/app/vaultDir2/secret.txt'; + vol.mkdirSync('/home/userData/appDir/app/vaultDir2', { recursive: true }); + vol.writeFileSync(siblingSecretPath, 'sibling secret'); + await expect( - files.write('/foo/bar/bytes1', new Uint8Array().buffer), - ).resolves.not.toThrow(); + files.write('../vaultDir2/secret.txt', getBufferFromText('overwritten')), + ).rejects.toThrow('Resolved path is out of root directory'); + expect(vol.readFileSync(siblingSecretPath, 'utf8')).toBe('sibling secret'); + }); + + test('Client cannot delete a file in a sibling directory with a matching root prefix', async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + const siblingSecretPath = '/home/userData/appDir/app/vaultDir2/secret.txt'; + vol.mkdirSync('/home/userData/appDir/app/vaultDir2', { recursive: true }); + vol.writeFileSync(siblingSecretPath, 'sibling secret'); + + await expect(files.delete(['../vaultDir2/secret.txt'])).rejects.toThrow( + 'Resolved path is out of root directory', + ); + expect(vol.readFileSync(siblingSecretPath, 'utf8')).toBe('sibling secret'); + }); + + test('Attempt to write content in directory must throws error', async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + await expect(files.write('/', new ArrayBuffer(1))).rejects.toThrowError( + 'Path is exist and it is not a file', + ); + }); + + test('File can be accessed inside root directory', async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + await expect(files.get('./secret.txt')).resolves.toStrictEqual(contentSample); + }); + + test('All ops respects the root limitations', async () => { + const files = new ElectronFilesController(storageApi, 'vaultDir'); + + await expect(files.delete(['../secret.txt'])).rejects.toThrowError( + 'Resolved path is out of root directory', + ); await expect( - files.write('/foo/bar/bytes2', new Uint8Array().buffer), - ).resolves.not.toThrow(); + files.write('../secret.txt', new Uint8Array().buffer), + ).rejects.toThrowError('Resolved path is out of root directory'); + + // Read file out of root directory + vol.writeFileSync('/home/userData/secret.txt', 'Secret data'); + await expect(files.get('../../secret.txt')).rejects.toThrowError( + 'Resolved path is out of root directory', + ); + }); +}); + +describe('FS backend', () => { + beforeAll(() => { + vol.reset(); + }); + + test('When backend is not started - any ops in controller throws error', async () => { + const filesController = new ElectronFilesController(storageApi, 'vaultDir'); + + await expect(filesController.list()).rejects.toThrow( + 'No handler registered for storage.list', + ); + await expect(filesController.get('/')).rejects.toThrow( + 'No handler registered for storage.createReadSession', + ); + await expect(filesController.write('/', new ArrayBuffer(1))).rejects.toThrow( + 'No handler registered for storage.createUploadSession', + ); + await expect(filesController.delete(['/'])).rejects.toThrow( + 'No handler registered for storage.delete', + ); + + // Setup handler + const storageHandlerCleanup = enableStorage(); + onTestFinished(storageHandlerCleanup); + + await expect(filesController.list()).resolves.toStrictEqual([]); + + // Clear handler again + storageHandlerCleanup(); + await expect(filesController.list()).rejects.toThrow( + 'No handler registered for storage.list', + ); + }); +}); + +describe('Many clients', () => { + const cleanups = [] as (() => void)[]; + beforeAll(() => { + vol.reset(); + cleanups.push(enableStorage()); + }); + afterAll(() => { + cleanups.forEach((cleanup) => cleanup()); + }); + + test('Files is scoped by client root directory', async () => { + const files1 = new ElectronFilesController(storageApi, 'dir1'); + const files2 = new ElectronFilesController(storageApi, 'dir2'); + + // Files list is isolated + await files1.write('test.bytes', new Uint8Array(1).buffer); + + await expect(files1.list()).resolves.toEqual(['/test.bytes']); + await expect(files2.list()).resolves.toEqual([]); + + // Each client can have different files with the same name + await files2.write('test.bytes', new Uint8Array(2).buffer); + + await expect(files1.get('test.bytes')).resolves.toStrictEqual( + new Uint8Array(1).buffer, + ); + await expect(files2.get('test.bytes')).resolves.toStrictEqual( + new Uint8Array(2).buffer, + ); }); +}); + +describe('Deletion tests', () => { + const cleanups = [] as (() => void)[]; + beforeAll(() => { + cleanups.push(enableStorage()); + }); + afterAll(() => { + cleanups.forEach((cleanup) => cleanup()); + }); + + beforeEach(() => { + vol.reset(); + }); + + test('One file can be deleted', async () => { + const files = new ElectronFilesController(storageApi, 'deletionTests'); + await createFiles(files, ['/bytes1', '/bytes2', '/bytes3']); - test('Single file may be deleted', async () => { await expect(files.list()).resolves.toContain('/bytes1'); + await files.delete(['bytes1']); await expect(files.list()).resolves.not.toContain('/bytes1'); }); - test('Few files may be deleted', async () => { + test('Few files can be deleted', async () => { + const files = new ElectronFilesController(storageApi, 'deletionTests'); + await createFiles(files, ['/bytes1', '/bytes2', '/bytes3']); + await expect(files.list()).resolves.toContain('/bytes2'); await expect(files.list()).resolves.toContain('/bytes3'); @@ -123,106 +368,66 @@ describe('Deletion tests', () => { }); test('Directory deletion deletes all its content', async () => { + const files = new ElectronFilesController(storageApi, 'deletionTests'); + await createFiles(files, ['/foo/bytes1', '/foo/bar/bytes1', '/foo/bar/bytes2']); + await expect(files.list()).resolves.toContain('/foo/bar/bytes1'); await expect(files.list()).resolves.toContain('/foo/bar/bytes2'); await files.delete(['/foo/bar']); await expect(files.list()).resolves.not.toContain('/foo/bar/bytes1'); await expect(files.list()).resolves.not.toContain('/foo/bar/bytes2'); + + await expect(files.list()).resolves.toContain('/foo/bytes1'); }); test('Root directory may be deleted', async () => { - await expect(files.list()).resolves.not.toEqual([]); + const files = new ElectronFilesController(storageApi, 'deletionTests'); + await createFiles(files, ['/foo/bytes1', '/foo/bar/bytes1', '/foo/bar/bytes2']); + + await expect(files.list()).resolves.toHaveLength(3); await files.delete(['/']); - await expect(files.list()).resolves.toEqual([]); + await expect(files.list()).resolves.toHaveLength(0); }); }); -describe('Many clients', () => { - const files1 = new ElectronFilesController(storageApi, 'vault1'); - const files2 = new ElectronFilesController(storageApi, 'vault2'); - - test('Available files is scoped by client root', async () => { - await files1.write('test.bytes', new Uint8Array(1).buffer); - - await expect(files1.list()).resolves.toEqual(['/test.bytes']); - await expect(files2.list()).resolves.toEqual([]); - - await files2.write('test.bytes', new Uint8Array(2).buffer); - - await expect(files1.get('test.bytes')).resolves.toStrictEqual( - new Uint8Array(1).buffer, - ); - await expect(files2.get('test.bytes')).resolves.toStrictEqual( - new Uint8Array(2).buffer, - ); +describe('Resources must not leak', () => { + beforeAll(() => { + vi.clearAllMocks(); }); - - test("Client can't traverse out of it's root", async () => { - await expect(files1.get('../vault2/test.bytes')).rejects.toThrowError( - 'Resolved path is out of root directory', - ); + afterAll(() => { + vi.clearAllMocks(); + }); + beforeEach(() => { + vol.reset(); }); -}); -test('Path traversal out of scope directory must not be possible', async () => { - await expect(filesController.get('/')).resolves.toBeNull(); - await expect(filesController.write('/', new ArrayBuffer(1))).rejects.toThrowError( - "EISDIR: illegal operation on a directory, open '/home/userData/appDir/app/vaultDir'", - ); + test('Reader must close session for successful reads', async () => { + const backend = createStorageBackend(); + onTestFinished(storageChannel.server(ipcMainHandler, backend)); - // Just to ensure we can read available files - vol.writeFileSync('/home/userData/appDir/app/vaultDir/secret.txt', 'Secret data'); - await expect(filesController.get('./secret.txt')).resolves.toBeInstanceOf( - ArrayBuffer, - ); + const files = new ElectronFilesController(storageApi, '/'); + await createFiles(files, ['/data']); - // Read file out of scoped directory - // Scope is defined by a client - vol.writeFileSync('/home/userData/appDir/secret.txt', 'Secret data'); - await expect(filesController.get('../secret.txt')).rejects.toThrowError( - 'Resolved path is out of root directory', - ); + const closeReaderSpy = vi.spyOn(backend, 'closeReader'); + await expect(files.get('/data')).resolves.not.toThrow(); + expect(closeReaderSpy).toHaveBeenCalledTimes(1); + }); - // All ops respects the limitation - await expect(filesController.delete(['../secret.txt'])).rejects.toThrowError( - 'Resolved path is out of root directory', - ); - await expect( - filesController.write('../secret.txt', new Uint8Array().buffer), - ).rejects.toThrowError('Resolved path is out of root directory'); - - // Read file out of root directory - vol.writeFileSync('/home/userData/secret.txt', 'Secret data'); - await expect(filesController.get('../../secret.txt')).rejects.toThrowError( - 'Resolved path is out of root directory', - ); -}); + test('Reader must close session when error occurs', async () => { + const backend = createStorageBackend(); + onTestFinished(storageChannel.server(ipcMainHandler, backend)); -test('When storage is disposed, any ops in controller throws error', async () => { - // Clean handler - storageHandlerCleanup(); - await expect(filesController.list()).rejects.toThrow( - 'No handler registered for storage.list', - ); - await expect(filesController.get('/')).rejects.toThrow( - 'No handler registered for storage.get', - ); - await expect(filesController.write('/', new ArrayBuffer(1))).rejects.toThrow( - 'No handler registered for storage.upload', - ); - await expect(filesController.delete(['/'])).rejects.toThrow( - 'No handler registered for storage.delete', - ); + const files = new ElectronFilesController(storageApi, '/'); + await createFiles(files, ['/data']); - // Setup handler - const storageHandlerCleanup2 = enableStorage(); - await expect(filesController.list()).resolves.toContainEqual('/test.txt'); + const closeReaderSpy = vi.spyOn(backend, 'closeReader'); - // Clear handler again - storageHandlerCleanup2(); - await expect(filesController.list()).rejects.toThrow( - 'No handler registered for storage.list', - ); + vi.spyOn(backend, 'readChunk').mockRejectedValueOnce( + new Error('Emulated read error'), + ); + await expect(files.get('/data')).rejects.toThrow('Emulated read error'); + expect(closeReaderSpy).toHaveBeenCalledTimes(1); + }); }); diff --git a/packages/app/src/electron/requests/storage/FileReadSession.ts b/packages/app/src/electron/requests/storage/FileReadSession.ts new file mode 100644 index 000000000..01ce9594d --- /dev/null +++ b/packages/app/src/electron/requests/storage/FileReadSession.ts @@ -0,0 +1,61 @@ +import { FileHandle, open } from 'node:fs/promises'; + +export class FileReadSession { + private closed = false; + private file: { + fd: FileHandle; + offset: number; + } | null = null; + public async open(filename: string) { + const fd = await open(filename, 'r'); + + this.file = { fd, offset: 0 }; + this.closed = false; + } + + public async size() { + const state = this.getState(); + const { size } = await state.fd.stat(); + return size; + } + + public async read(size: number): Promise { + const state = this.getState(); + + const buffer = new ArrayBuffer(size); + const { bytesRead } = await state.fd.read( + new Uint8Array(buffer), + 0, + size, + state.offset, + ); + + state.offset += bytesRead; + + // No data + if (bytesRead === 0) { + // await this.close(); + return null; + } + + return buffer.slice(0, bytesRead); + } + + public async close(): Promise { + if (this.closed) return; + + this.closed = true; + if (this.file) { + const state = this.file; + this.file = null; + await state.fd.close(); + } + } + + public getState() { + if (this.closed) throw new Error('File descriptor is closed'); + if (!this.file) throw new Error('File is not opened'); + + return this.file; + } +} diff --git a/packages/app/src/electron/requests/storage/FileReadSessions.ts b/packages/app/src/electron/requests/storage/FileReadSessions.ts new file mode 100644 index 000000000..4aa5a56f7 --- /dev/null +++ b/packages/app/src/electron/requests/storage/FileReadSessions.ts @@ -0,0 +1,56 @@ +import { existsSync, renameSync, statSync } from 'fs'; + +import { FileReadSession } from './FileReadSession'; +import { PathsResolver } from './PathsResolver'; + +export class FileReadSessions { + private readonly sessions = new Map(); + + constructor(private readonly paths: PathsResolver) {} + + public async create(fileId: string, subdir: string) { + const paths = this.paths.getFilePaths(subdir, fileId); + + if (!this.paths.isAllowedPath(paths.resolvedPath)) return null; + + // Restore file + if ( + !existsSync(paths.resolvedPath) && + existsSync(paths.bkp) && + statSync(paths.bkp).isFile() + ) { + renameSync(paths.bkp, paths.resolvedPath); + } + + // Exist for non exists files + if (!existsSync(paths.resolvedPath) || !statSync(paths.resolvedPath).isFile()) + return null; + + const session = new FileReadSession(); + await session.open(paths.resolvedPath); + + const sessionId = this.getSessionId(); + this.sessions.set(sessionId, session); + + return sessionId; + } + + public getById(sessionId: string): FileReadSession | null { + return this.sessions.get(sessionId) ?? null; + } + + public async close(sessionId: string): Promise { + const session = this.sessions.get(sessionId); + + if (!session) return; + + this.sessions.delete(sessionId); + + await session.close(); + } + + private sessionId = 0; + private getSessionId(): string { + return String(++this.sessionId); + } +} diff --git a/packages/app/src/electron/requests/storage/FilesStorage.ts b/packages/app/src/electron/requests/storage/FilesStorage.ts new file mode 100644 index 000000000..a20014433 --- /dev/null +++ b/packages/app/src/electron/requests/storage/FilesStorage.ts @@ -0,0 +1,70 @@ +import { existsSync, rmSync, statSync } from 'fs'; +import path from 'path'; + +import { readdir, rm } from 'fs/promises'; +import { PathsResolver } from './PathsResolver'; +import { UploadSessions } from './UploadSessions'; + +export class FilesStorage { + constructor( + private readonly paths: PathsResolver, + private readonly opSessions: UploadSessions, + ) {} + + async delete(fileIds: string[], subdir: string) { + for (const fileId of fileIds) { + const { resolvedPath, bkp, tmp } = this.paths.getFilePaths(subdir, fileId); + + // Skip special files + if (!this.paths.isAllowedPath(resolvedPath)) continue; + + if (existsSync(resolvedPath) && statSync(resolvedPath).isDirectory()) { + // Recursive directory deletion + await rm(resolvedPath, { force: true, recursive: true }); + console.debug('Directory removed', resolvedPath); + } else { + // Delete file and its related files + const pathsToRemove = [resolvedPath, bkp]; + if (!this.opSessions.getByFilename(resolvedPath)) pathsToRemove.push(tmp); + + for (const path of pathsToRemove) { + if (!existsSync(path) || !statSync(path).isFile()) continue; + rmSync(path); + console.debug('File deleted', path); + } + } + } + } + + async list(subdir: string) { + const filesDir = this.paths.getScopedPath(subdir); + + if (!existsSync(filesDir)) return []; + + const files: string[] = []; + for (const entry of await readdir(filesDir, { + recursive: true, + withFileTypes: true, + })) { + // Skip non-files + if (!entry.isFile() || entry.isSymbolicLink()) continue; + + const resolvedFilename = path.resolve( + path.join(entry.parentPath, entry.name), + ); + + // Skip special files + if (!this.paths.isAllowedPath(resolvedFilename)) continue; + + // Remove root path + const virtualPath = resolvedFilename + .slice(filesDir.length) + .split(path.sep) + .join('/'); + + files.push(virtualPath); + } + + return files; + } +} diff --git a/packages/app/src/electron/requests/storage/MutexMap.ts b/packages/app/src/electron/requests/storage/MutexMap.ts new file mode 100644 index 000000000..9163f4389 --- /dev/null +++ b/packages/app/src/electron/requests/storage/MutexMap.ts @@ -0,0 +1,41 @@ +import { Mutex, MutexInterface } from 'async-mutex'; + +export class MutexMap { + private readonly mutexMap = new Map< + K, + { + mutex: Mutex; + users: number; + } + >(); + + private getMutexById(id: K) { + const state = this.mutexMap.get(id); + if (state) return state; + + const newState = { + mutex: new Mutex(), + users: 0, + }; + this.mutexMap.set(id, newState); + + return newState; + } + + public runExclusive(id: K, callback: MutexInterface.Worker): Promise { + const state = this.getMutexById(id); + + state.users++; + return state.mutex.runExclusive(callback).finally(() => { + state.users--; + if ( + state.users > 0 || + state.mutex.isLocked() || + this.mutexMap.get(id) !== state + ) + return; + + this.mutexMap.delete(id); + }); + } +} diff --git a/packages/app/src/electron/requests/storage/PathsResolver.ts b/packages/app/src/electron/requests/storage/PathsResolver.ts new file mode 100644 index 000000000..7990b4085 --- /dev/null +++ b/packages/app/src/electron/requests/storage/PathsResolver.ts @@ -0,0 +1,67 @@ +import path from 'node:path'; + +import { getUserDataPath, joinPath } from '@electron/utils/files'; + +export type StorageOptions = { tmpPrefix?: string }; + +export type FilePaths = { + resolvedPath: string; + dirname: string; + filename: string; + tmp: string; + bkp: string; +}; + +export class PathsResolver { + private readonly config; + constructor(config: StorageOptions = {}) { + this.config = { + tmpPrefix: '.tmp-fs', + ...config, + }; + } + + public getScopedPath(subdir: string | undefined, path?: string) { + return path ? joinPath(getUserDataPath(subdir), path) : getUserDataPath(subdir); + } + + public getFilePaths(subdir: string, fileId: string) { + const { tmpPrefix } = this.config; + + const resolvedPath = this.getScopedPath(subdir, fileId); + const dirname = path.dirname(resolvedPath); + const filename = path.basename(resolvedPath); + + const tmp = path.resolve( + path.join(dirname, [tmpPrefix, 'tmp', filename].join('-')), + ); + if (!tmp.startsWith(dirname)) + throw new Error('Temp file path is out of allowed path'); + + const bkp = path.resolve( + path.join(dirname, [tmpPrefix, 'bkp', filename].join('-')), + ); + if (!bkp.startsWith(dirname)) + throw new Error('Backup file path is out of allowed path'); + + return { + resolvedPath, + dirname, + filename, + tmp, + bkp, + } satisfies FilePaths; + } + + public isAllowedPath(filename: string) { + const { tmpPrefix } = this.config; + + return filename + .split(path.sep) + .every((segment) => !segment.startsWith(tmpPrefix)); + } + + public getPrefix() { + return this.config.tmpPrefix; + } +} diff --git a/packages/app/src/electron/requests/storage/Tombstones.ts b/packages/app/src/electron/requests/storage/Tombstones.ts new file mode 100644 index 000000000..daea45b22 --- /dev/null +++ b/packages/app/src/electron/requests/storage/Tombstones.ts @@ -0,0 +1,61 @@ +export class Tombstones { + private readonly entries = new Map< + K, + { value: V; expiresAt: number; timer: ReturnType } + >(); + + constructor(private readonly timeoutMs: number) { + if (!Number.isFinite(timeoutMs) || timeoutMs < 0) + throw new Error('Tombstone timeout must be a non-negative number'); + } + + public set(key: K, value: V) { + this.delete(key); + + const entry = { + value, + expiresAt: Date.now() + this.timeoutMs, + timer: undefined as unknown as ReturnType, + }; + + entry.timer = setTimeout(() => this.expire(key, entry), this.timeoutMs); + this.entries.set(key, entry); + } + + public get(key: K): V | undefined { + const entry = this.entries.get(key); + if (!entry) return undefined; + + // Also enforce expiry if the timer callback has not run yet. + if (entry.expiresAt <= Date.now()) { + this.expire(key, entry); + return undefined; + } + + return entry.value; + } + + public delete(key: K) { + const entry = this.entries.get(key); + if (!entry) return false; + + clearTimeout(entry.timer); + return this.entries.delete(key); + } + + public clear() { + for (const entry of this.entries.values()) clearTimeout(entry.timer); + this.entries.clear(); + } + + private expire( + key: K, + expected: { value: V; expiresAt: number; timer: ReturnType }, + ) { + // Don't let an old timer remove a newer tombstone for the same key. + if (this.entries.get(key) !== expected) return; + + clearTimeout(expected.timer); + this.entries.delete(key); + } +} diff --git a/packages/app/src/electron/requests/storage/UploadSession.ts b/packages/app/src/electron/requests/storage/UploadSession.ts new file mode 100644 index 000000000..842d8ef83 --- /dev/null +++ b/packages/app/src/electron/requests/storage/UploadSession.ts @@ -0,0 +1,128 @@ +/* eslint-disable @typescript-eslint/no-use-before-define */ +import { once } from 'node:events'; +import { + createWriteStream, + existsSync, + renameSync, + rmSync, + statSync, + WriteStream, +} from 'node:fs'; +import { mkdir } from 'node:fs/promises'; + +import { FilePaths } from './PathsResolver'; + +export class UploadSession { + constructor(private readonly paths: FilePaths) {} + + private stream?: WriteStream; + private isInitializationCalled = false; + private onCommit?: () => void; + public async init(onCommit?: () => void) { + this.assertNotAborted(); + + if (this.stream) throw new Error('Session is already initialized'); + if (this.isInitializationCalled) throw new Error('Initialization is in progress'); + this.isInitializationCalled = true; + this.onCommit = onCommit; + + const { resolvedPath, dirname, tmp } = this.paths; + + // Prevent write into directory + if (existsSync(resolvedPath) && !statSync(resolvedPath).isFile()) + throw new Error(`Path is exist and it is not a file '${resolvedPath}'`); + + // Ensure directory + await mkdir(dirname, { recursive: true }); + + // Open stream + if (existsSync(tmp)) rmSync(tmp); + const stream = createWriteStream(tmp); + await once(stream, 'open'); + + this.stream = stream; + } + + private error?: unknown; + public async abort(error: unknown) { + if (this.error !== undefined) return; + + // Set error immediately + this.error = error; + + // Close stream + if (this.stream) { + this.stream.close(); + await once(this.stream, 'close'); + } + } + + public async write(buffer: ArrayBuffer) { + this.assertNotAborted(); + + if (!this.stream) throw new Error('Session is not initialized yet'); + + if (!this.stream.write(new Uint8Array(buffer))) { + const stream = this.stream; + await new Promise((resolve, reject) => { + const cleanup = () => { + stream.off('drain', onDrain); + stream.off('close', onClose); + stream.off('error', onError); + }; + const onDrain = () => { + cleanup(); + resolve(); + }; + const onClose = () => { + cleanup(); + reject(this.error ?? new Error('Upload stream closed before drain')); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + + stream.once('drain', onDrain); + stream.once('close', onClose); + stream.once('error', onError); + }); + } + } + + public async commit() { + this.assertNotAborted(); + + if (!this.stream) throw new Error('Session is not initialized yet'); + + const stream = this.stream; + const paths = this.paths; + + try { + // Finish stream + stream.end(); + await once(stream, 'finish'); + this.assertNotAborted(); + + // Make sure file is uploaded + if (!existsSync(paths.tmp)) throw new Error('Temporary file is not found'); + + // Remove backup file + if (existsSync(paths.bkp)) rmSync(paths.bkp); + + // Backup file + if (existsSync(paths.resolvedPath)) renameSync(paths.resolvedPath, paths.bkp); + + // Rename temp file + renameSync(paths.tmp, paths.resolvedPath); + if (existsSync(paths.bkp)) rmSync(paths.bkp); + } finally { + this.onCommit?.(); + } + } + + public assertNotAborted() { + // eslint-disable-next-line @typescript-eslint/only-throw-error + if (this.error !== undefined) throw this.error; + } +} diff --git a/packages/app/src/electron/requests/storage/UploadSessions.ts b/packages/app/src/electron/requests/storage/UploadSessions.ts new file mode 100644 index 000000000..46eca0a10 --- /dev/null +++ b/packages/app/src/electron/requests/storage/UploadSessions.ts @@ -0,0 +1,74 @@ +import { MutexMap } from './MutexMap'; +import { PathsResolver } from './PathsResolver'; +import { Tombstones } from './Tombstones'; +import { UploadSession } from './UploadSession'; + +export class UploadSessions { + private readonly pathUploadSessions = new Map(); + private readonly uploadSessions = new Map(); + private readonly mutexMap = new MutexMap(); + private readonly tombstones = new Tombstones(60_000); + + constructor(private readonly paths: PathsResolver) {} + + public async create(fileId: string, subdir: string) { + const paths = this.paths.getFilePaths(subdir, fileId); + + if (!this.paths.isAllowedPath(paths.resolvedPath)) + throw new Error(`File name must not start from '${this.paths.getPrefix()}'`); + + return this.mutexMap.runExclusive(paths.resolvedPath, async () => { + // Cancel previous session + const previousSessionId = this.pathUploadSessions.get(paths.resolvedPath); + if (previousSessionId !== undefined) { + const session = this.uploadSessions.get(previousSessionId); + if (session) { + const error = new Error('Another session is started'); + this.tombstones.set(previousSessionId, error); + this.uploadSessions.delete(previousSessionId); + await session.abort(error); + } + } + + // Start new session + const session = new UploadSession(paths); + + const sessionId = this.getSessionId(); + this.pathUploadSessions.set(paths.resolvedPath, sessionId); + this.uploadSessions.set(sessionId, session); + + await session.init(() => { + // Terminate session + this.uploadSessions.delete(sessionId); + + // Cleanup related + const currentSessionId = this.pathUploadSessions.get(paths.resolvedPath); + if (currentSessionId === sessionId) + this.pathUploadSessions.delete(paths.resolvedPath); + }); + + return sessionId; + }); + } + + public getByFilename(filename: string) { + const sessionId = this.pathUploadSessions.get(filename); + return sessionId ? this.getById(sessionId) : undefined; + } + + public getById(sessionId: string) { + const session = this.uploadSessions.get(sessionId); + if (session) return session; + + const tombstone = this.tombstones.get(sessionId); + // eslint-disable-next-line @typescript-eslint/only-throw-error + if (tombstone) throw tombstone; + + return null; + } + + private uploadId = 0; + private getSessionId() { + return String(++this.uploadId); + } +} diff --git a/packages/app/src/electron/requests/storage/__snapshots__/ElectronFilesController.integration.test.ts.snap b/packages/app/src/electron/requests/storage/__snapshots__/ElectronFilesController.integration.test.ts.snap index 825302c5f..0c0089433 100644 --- a/packages/app/src/electron/requests/storage/__snapshots__/ElectronFilesController.integration.test.ts.snap +++ b/packages/app/src/electron/requests/storage/__snapshots__/ElectronFilesController.integration.test.ts.snap @@ -1,36 +1,36 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`Check files content > Files content 1`] = ` -[ - { - "content": "File text content", - "path": "/test.txt", - }, - { - "content": "File text content", - "path": "/foo/bar/baz/test.txt", - }, -] -`; - -exports[`Write files > Controller files list after writing 1`] = ` +exports[`Basic ops > File uploading persist data on disk > Controller files list after writing 1`] = ` [ "/test.txt", "/foo/bar/baz/test.txt", ] `; -exports[`Write files > FS structure after writing 1`] = ` +exports[`Basic ops > File uploading persist data on disk > FS structure after writing 1`] = ` [ - "/home/userData/appDir/app", + "/home", + "/home/userData", "/home/userData/appDir", + "/home/userData/appDir/app", + "/home/userData/appDir/app/vaultDir", + "/home/userData/appDir/app/vaultDir/foo", "/home/userData/appDir/app/vaultDir/foo/bar", "/home/userData/appDir/app/vaultDir/foo/bar/baz", - "/home/userData/appDir/app/vaultDir/foo", - "/home", - "/home/userData/appDir/app/vaultDir/test.txt", "/home/userData/appDir/app/vaultDir/foo/bar/baz/test.txt", - "/home/userData", - "/home/userData/appDir/app/vaultDir", + "/home/userData/appDir/app/vaultDir/test.txt", +] +`; + +exports[`Basic ops > File uploading persist data on disk > Files content 1`] = ` +[ + { + "content": "File text content", + "path": "/test.txt", + }, + { + "content": "File text content", + "path": "/foo/bar/baz/test.txt", + }, ] `; diff --git a/packages/app/src/electron/requests/storage/index.ts b/packages/app/src/electron/requests/storage/index.ts index 59069ee94..8e1c36d15 100644 --- a/packages/app/src/electron/requests/storage/index.ts +++ b/packages/app/src/electron/requests/storage/index.ts @@ -1,8 +1,19 @@ import { createChannel } from '../../utils/ipc'; export type StorageChannelAPI = { - upload: (id: string, buffer: ArrayBuffer, subdir: string) => Promise; - get: (id: string, subdir: string) => Promise; + // File write + createUploadSession: (fileId: string, subdir: string) => Promise; + uploadChunk: (sessionId: string, buffer: ArrayBuffer) => Promise; + commitUpload: (sessionId: string) => Promise; + + // File reads + createReadSession: ( + fileId: string, + subdir: string, + ) => Promise<{ id: string; size: number } | null>; + readChunk: (sessionId: string, size: number) => Promise; + closeReader: (sessionId: string) => Promise; + delete: (ids: string[], subdir: string) => Promise; list: (subdir: string) => Promise; }; diff --git a/packages/app/src/electron/requests/storage/main.ts b/packages/app/src/electron/requests/storage/main.ts index 08c085942..2eeadd5b6 100644 --- a/packages/app/src/electron/requests/storage/main.ts +++ b/packages/app/src/electron/requests/storage/main.ts @@ -1,65 +1,102 @@ import { IpcMainInvokeEvent } from 'electron'; -import { existsSync, statSync } from 'fs'; -import path from 'path'; -import recursive from 'recursive-readdir'; import { ApiToHandlers } from '@electron/utils/ipc'; -import { recoveryAtomicFile, writeFileAtomic } from '@utils/files'; -import { getUserDataPath, joinPath } from '../../utils/files'; import { ipcMainHandler } from '../../utils/ipc/ipcMainHandler'; -import { mkdir, readFile, rm } from 'fs/promises'; +import { FileReadSessions } from './FileReadSessions'; +import { FilesStorage } from './FilesStorage'; +import { PathsResolver, StorageOptions } from './PathsResolver'; +import { UploadSessions } from './UploadSessions'; import { storageChannel, StorageChannelAPI } from '.'; -export const createStorageBackend = () => { - const getScopedPath = (subdir: string | undefined, path?: string) => - path ? joinPath(getUserDataPath(subdir), path) : getUserDataPath(subdir); +export const createStorageBackend = ({ tmpPrefix = '.tmp-fs' }: StorageOptions = {}) => { + const pathsResolver = new PathsResolver({ tmpPrefix }); + const uploadSessions = new UploadSessions(pathsResolver); + const storage = new FilesStorage(pathsResolver, uploadSessions); + + const readSessions = new FileReadSessions(pathsResolver); return { - async upload({ req: [id, buffer, subdir] }) { - const filePath = getScopedPath(subdir, id); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFileAtomic(filePath, Buffer.from(new Uint8Array(buffer))); + async delete({ req: [fileIds, subdir] }) { + return storage.delete(fileIds, subdir); }, - async get({ req: [id, subdir] }) { - const filePath = getScopedPath(subdir, id); + async list({ req: [subdir] }) { + return storage.list(subdir); + }, - recoveryAtomicFile(filePath); + createUploadSession: async ({ + req: [fileId, subdir], + }: { + req: [id: string, subdir: string]; + ctx: Electron.IpcMainInvokeEvent; + }): Promise => { + return uploadSessions.create(fileId, subdir); + }, - if (!existsSync(filePath) || !statSync(filePath).isFile()) return null; + uploadChunk: async function ({ + req: [sessionId, buffer], + }: { + req: [id: string, buffer: ArrayBuffer]; + ctx: Electron.IpcMainInvokeEvent; + }) { + const session = uploadSessions.getById(sessionId); + if (!session) throw new Error(`No session found with id ${sessionId}`); - const buffer = await readFile(filePath); - return new Uint8Array(buffer).buffer; + await session.write(buffer); }, - async delete({ req: [ids, subdir] }) { - for (const id of ids) { - const filePath = getScopedPath(subdir, id); - - if (!existsSync(filePath)) { - console.debug('Not found file', filePath); - continue; - } + commitUpload: async function ({ + req: [sessionId], + }: { + req: [id: string]; + ctx: Electron.IpcMainInvokeEvent; + }) { + const session = uploadSessions.getById(sessionId); + if (!session) throw new Error(`No session found with id ${sessionId}`); - await rm(filePath, { force: true, recursive: true }); - console.debug('Removed file', filePath); - } + await session.commit(); }, - async list({ req: [subdir] }) { - const filesDir = getScopedPath(subdir); + createReadSession: async function ({ + req: [fileId, subdir], + }: { + req: [fileId: string, subdir: string]; + ctx: Electron.IpcMainInvokeEvent; + }) { + const sessionId = await readSessions.create(fileId, subdir); + if (sessionId === null) return null; + + const session = readSessions.getById(sessionId); + if (!session) throw new Error(`No session found for id ${sessionId}`); - if (!existsSync(filesDir)) return []; + const size = await session.size(); - const files = await recursive(filesDir); - return files.map((path) => - // Remove root path - path.slice(filesDir.length), - ); + return { id: sessionId, size }; + }, + readChunk: async function ({ + req: [sessionId, size], + }: { + req: [sessionId: string, size: number]; + ctx: Electron.IpcMainInvokeEvent; + }) { + const session = readSessions.getById(sessionId); + if (!session) throw new Error(`No session found for id ${sessionId}`); + + return session.read(size); + }, + closeReader: async function ({ + req: [sessionId], + }: { + req: [sessionId: string]; + ctx: Electron.IpcMainInvokeEvent; + }) { + const session = readSessions.getById(sessionId); + if (!session) throw new Error(`No session found for id ${sessionId}`); + await session.close(); }, } satisfies ApiToHandlers; }; -export const enableStorage = () => - storageChannel.server(ipcMainHandler, createStorageBackend()); +export const enableStorage = (options: StorageOptions = {}) => + storageChannel.server(ipcMainHandler, createStorageBackend(options)); diff --git a/packages/app/src/electron/requests/storage/renderer.ts b/packages/app/src/electron/requests/storage/renderer.ts index 0610d348f..bdcea310f 100644 --- a/packages/app/src/electron/requests/storage/renderer.ts +++ b/packages/app/src/electron/requests/storage/renderer.ts @@ -1,4 +1,4 @@ -import { IEncryptionController } from '@core/encryption'; +import { BufferCursor } from '@core/encryption/utils/bytes/BufferCursor'; import { IFilesStorage } from '@core/features/files'; import { ipcRendererFetcher } from '../../utils/ipc/ipcRendererFetcher'; @@ -12,33 +12,60 @@ export class ElectronFilesController implements IFilesStorage { constructor( private readonly storageApi: StorageChannelAPI, private readonly subdirectory: string, - private readonly encryption?: IEncryptionController, - ) { - this.subdirectory = subdirectory; - this.encryption = encryption; - } + private readonly config: { chunkSize?: number } = {}, + ) {} public async write(filename: string, buffer: ArrayBuffer) { - const encryptedBuffer = this.encryption - ? await this.encryption.encrypt(buffer) - : buffer; + const sessionId = await this.storageApi.createUploadSession( + filename, + this.subdirectory, + ); + + const bufferCursor = new BufferCursor(buffer); + while (bufferCursor.getRemainingBytes() > 0) { + const slice = bufferCursor.readBytes( + 1024 ** 2 * (this.config.chunkSize ?? 5), + ); + if (!slice) throw new Error('Unexpected end of file'); - // Absolute file name with no slash - return this.storageApi.upload(filename, encryptedBuffer, this.subdirectory); + await this.storageApi.uploadChunk(sessionId, slice.slice(0).buffer); + } + + await this.storageApi.commitUpload(sessionId); } - public async get(id: string) { - return this.storageApi.get(id, this.subdirectory).then((buffer) => { - // Don't handle empty data - if (!buffer) return buffer; + public async get(filename: string) { + const sessionInfo = await this.storageApi.createReadSession( + filename, + this.subdirectory, + ); + if (!sessionInfo) return null; + + const { size, id: sessionId } = sessionInfo; + + const buffer = new Uint8Array(size); + try { + const chunkSize = 1024 ** 2 * (this.config.chunkSize ?? 5); + let offset = 0; + while (true) { + const chunk = await this.storageApi.readChunk(sessionId, chunkSize); + + // End when whole file is drained + if (!chunk) break; + + // Write + buffer.set(new Uint8Array(chunk), offset); + offset += chunk.byteLength; + } + } finally { + await this.storageApi.closeReader(sessionId); + } - if (!this.encryption) return buffer; - return this.encryption.decrypt(buffer); - }); + return buffer.buffer; } - public async delete(ids: string[]) { - return this.storageApi.delete(ids, this.subdirectory); + public async delete(filenames: string[]) { + return this.storageApi.delete(filenames, this.subdirectory); } public async list() { diff --git a/packages/app/src/electron/requests/storage/storage-backend.test.ts b/packages/app/src/electron/requests/storage/storage-backend.test.ts new file mode 100644 index 000000000..cda301b27 --- /dev/null +++ b/packages/app/src/electron/requests/storage/storage-backend.test.ts @@ -0,0 +1,288 @@ +import { vol } from 'memfs'; +import { getRandomBytes } from '@core/encryption/utils/random'; +import { readdir } from '@tests/utils/fs'; + +import { enableStorage } from './main'; +import { storageApi } from './renderer'; + +vi.mock('fs', () => vi.importActual('@mocks/fs')); +vi.mock('fs/promises', () => vi.importActual('@mocks/fs/promises')); +vi.mock('electron', () => vi.importActual('@mocks/electron')); + +describe('Special names', () => { + beforeEach(() => { + vol.reset(); + }); + + test('File name with special prefix cannot be created', async () => { + onTestFinished(enableStorage({ tmpPrefix: '.tmp' })); + + await expect(storageApi.createUploadSession('.tmp', '/foo/bar')).rejects.toThrow( + "File name must not start from '.tmp'", + ); + + await expect( + storageApi.createUploadSession('foo/bar/.tmp', '/foo/bar'), + ).rejects.toThrow("File name must not start from '.tmp'"); + + await expect( + storageApi.createUploadSession('foo/bar/.tmp-bkp', '/foo/bar'), + ).rejects.toThrow("File name must not start from '.tmp'"); + }); + + test('Files with special name is unavailable', async () => { + onTestFinished(enableStorage({ tmpPrefix: '.tmp' })); + + // Start uploading session + const session = await storageApi.createUploadSession('filename', '/foo/bar'); + await storageApi.uploadChunk(session, getRandomBytes(1024).buffer); + + // Temp file is present in FS + expect(readdir('/')).toContain( + '/home/userData/appDir/app/foo/bar/.tmp-tmp-filename', + ); + + await expect(storageApi.list('/')).resolves.toEqual([]); + await expect( + storageApi.createReadSession('.tmp-tmp-filename', '/foo/bar'), + ).resolves.toBe(null); + + // Deletion does not work + await expect( + storageApi.delete(['.tmp-tmp-filename'], '/foo/bar'), + ).resolves.toBeUndefined(); + expect(readdir('/')).toContain( + '/home/userData/appDir/app/foo/bar/.tmp-tmp-filename', + ); + }); +}); + +describe('Upload sessions', () => { + beforeEach(() => { + vol.reset(); + }); + + test('File can be loaded with session', async () => { + onTestFinished(enableStorage()); + + const session = await storageApi.createUploadSession('filename', '/foo/bar'); + + await storageApi.uploadChunk(session, getRandomBytes(1024).buffer); + await storageApi.uploadChunk(session, getRandomBytes(1024).buffer); + await storageApi.uploadChunk(session, getRandomBytes(1024).buffer); + + await storageApi.commitUpload(session); + + expect( + vol.readFileSync('/home/userData/appDir/app/foo/bar/filename'), + ).toHaveLength(1024 * 3); + }); + + test('Session cannot be re-used', async () => { + onTestFinished(enableStorage()); + + const session = await storageApi.createUploadSession('filename', '/foo/bar'); + await storageApi.uploadChunk(session, getRandomBytes(1024).buffer); + await storageApi.commitUpload(session); + + expect( + vol.readFileSync('/home/userData/appDir/app/foo/bar/filename'), + ).toHaveLength(1024); + + await expect( + storageApi.uploadChunk(session, getRandomBytes(1024).buffer), + ).rejects.toThrow('No session found'); + await expect(storageApi.commitUpload(session)).rejects.toThrow( + 'No session found', + ); + }); + + test('File can be changed', async () => { + onTestFinished(enableStorage()); + + // Write + const session = await storageApi.createUploadSession('filename', '/foo/bar'); + await storageApi.uploadChunk(session, getRandomBytes(1024).buffer); + await storageApi.uploadChunk(session, getRandomBytes(1024).buffer); + await storageApi.commitUpload(session); + + expect( + vol.readFileSync('/home/userData/appDir/app/foo/bar/filename'), + ).toHaveLength(1024 * 2); + + // Write + const session2 = await storageApi.createUploadSession('filename', '/foo/bar'); + await storageApi.uploadChunk(session2, getRandomBytes(100).buffer); + await storageApi.uploadChunk(session2, getRandomBytes(100).buffer); + await storageApi.commitUpload(session2); + + expect( + vol.readFileSync('/home/userData/appDir/app/foo/bar/filename'), + ).toHaveLength(100 * 2); + }); + + test('Creation of new session for the same path must cancel previous session', async () => { + onTestFinished(enableStorage()); + + const session1 = await storageApi.createUploadSession('filename', '/foo/bar'); + await storageApi.uploadChunk(session1, getRandomBytes(1024).buffer); + + const session2 = await storageApi.createUploadSession('filename', '/foo/bar'); + await expect( + storageApi.uploadChunk(session1, getRandomBytes(1024).buffer), + ).rejects.toThrow('Another session is started'); + await expect( + storageApi.uploadChunk(session2, getRandomBytes(1024).buffer), + ).resolves.not.toThrow(); + + await expect(storageApi.commitUpload(session1)).rejects.toThrow( + 'Another session is started', + ); + await expect(storageApi.commitUpload(session2)).resolves.not.toThrow(); + + expect( + vol.readFileSync('/home/userData/appDir/app/foo/bar/filename'), + ).toHaveLength(1024); + }); + + test('Aborted session reasons are retained temporarily, then discarded', async () => { + vi.useFakeTimers(); + onTestFinished(() => { + vi.useRealTimers(); + }); + onTestFinished(enableStorage()); + + const previousSession = await storageApi.createUploadSession( + 'filename', + '/foo/bar', + ); + await storageApi.uploadChunk(previousSession, getRandomBytes(1024).buffer); + + const currentSession = await storageApi.createUploadSession( + 'filename', + '/foo/bar', + ); + await storageApi.commitUpload(currentSession); + + await expect( + storageApi.uploadChunk(previousSession, getRandomBytes(1024).buffer), + ).rejects.toThrow('Another session is started'); + + await vi.advanceTimersByTimeAsync(59_000); + await expect( + storageApi.uploadChunk(previousSession, getRandomBytes(1024).buffer), + ).rejects.toThrow('Another session is started'); + + await vi.advanceTimersByTimeAsync(10_000); + + await expect( + storageApi.uploadChunk(previousSession, getRandomBytes(1024).buffer), + ).rejects.toThrow('No session found'); + }); + + test('No race conditions for sessions', async () => { + onTestFinished(enableStorage()); + + const sessionIds = await Promise.all([ + storageApi.createUploadSession('filename', '/foo/bar'), + storageApi.createUploadSession('filename', '/foo/bar'), + storageApi.createUploadSession('filename', '/foo/bar'), + storageApi.createUploadSession('filename', '/foo/bar'), + storageApi.createUploadSession('filename', '/foo/bar'), + ]); + + await expect( + Promise.allSettled( + sessionIds.map((id) => + storageApi.uploadChunk(id, getRandomBytes(100).buffer), + ), + ).then((results) => + results.filter((result) => result.status === 'fulfilled'), + ), + ).resolves.toHaveLength(1); + }); + + test('Many sessions may exists in parallel', async () => { + onTestFinished(enableStorage()); + + const session1 = await storageApi.createUploadSession('filename1', '/foo/bar'); + const session2 = await storageApi.createUploadSession('filename2', '/foo/bar'); + + await storageApi.uploadChunk(session1, getRandomBytes(1024).buffer); + await storageApi.uploadChunk(session2, getRandomBytes(1024).buffer); + + await storageApi.uploadChunk(session1, getRandomBytes(1024).buffer); + await storageApi.uploadChunk(session2, getRandomBytes(1024).buffer); + + await expect(storageApi.commitUpload(session1)).resolves.not.toThrow(); + await expect(storageApi.commitUpload(session2)).resolves.not.toThrow(); + + expect( + vol.readFileSync('/home/userData/appDir/app/foo/bar/filename1'), + ).toHaveLength(1024 * 2); + expect( + vol.readFileSync('/home/userData/appDir/app/foo/bar/filename2'), + ).toHaveLength(1024 * 2); + }); +}); + +describe('File integrity', () => { + beforeEach(() => { + vol.reset(); + }); + + test('File can be restored from backup', async () => { + onTestFinished(enableStorage({ tmpPrefix: '.tmp' })); + + const fileContent = getRandomBytes(1024); + + // Create backup file + vol.mkdirSync('/home/userData/appDir/app/foo/bar', { recursive: true }); + vol.writeFileSync( + '/home/userData/appDir/app/foo/bar/.tmp-bkp-filename', + fileContent, + ); + + const session = await storageApi.createReadSession('filename', '/foo/bar'); + expect(session).not.toBe(null); + await expect(storageApi.readChunk(session!.id, 10_000)).resolves.toStrictEqual( + fileContent.buffer, + ); + }); + + test('File deletes with all related tmp files', async () => { + onTestFinished(enableStorage({ tmpPrefix: '.tmp' })); + + const fileContent = getRandomBytes(1024); + + // Create backup file + vol.mkdirSync('/home/userData/appDir/app/foo/bar', { recursive: true }); + vol.writeFileSync( + '/home/userData/appDir/app/foo/bar/.tmp-bkp-filename', + fileContent, + ); + vol.writeFileSync('/home/userData/appDir/app/foo/bar/filename', fileContent); + expect(readdir('/')).toContainEqual(expect.stringContaining('filename')); + + await storageApi.delete(['filename'], '/foo/bar'); + expect(readdir('/')).not.toContainEqual(expect.stringContaining('filename')); + }); + + test('Deleting a file removes its backup even when the target is missing', async () => { + onTestFinished(enableStorage({ tmpPrefix: '.tmp' })); + + const root = '/home/userData/appDir/app/foo/bar'; + const backupPath = `${root}/.tmp-bkp-note`; + vol.mkdirSync(root, { recursive: true }); + vol.writeFileSync(backupPath, 'deleted content'); + expect(vol.existsSync(`${root}/note`)).toBe(false); + expect(vol.existsSync(backupPath)).toBe(true); + + await storageApi.delete(['note'], '/foo/bar'); + + expect(vol.existsSync(backupPath)).toBe(false); + await expect(storageApi.createReadSession('note', '/foo/bar')).resolves.toBe( + null, + ); + }); +}); diff --git a/packages/app/src/electron/utils/files.ts b/packages/app/src/electron/utils/files.ts index 49c0e235a..e179e5046 100644 --- a/packages/app/src/electron/utils/files.ts +++ b/packages/app/src/electron/utils/files.ts @@ -3,14 +3,22 @@ import path from 'path'; import { isDevMode } from './app'; +export const isRootedPath = (root: string, targetPath: string, separator = path.sep) => { + const pathSegments = targetPath.split(separator); + return root + .split(separator) + .every((rootSegment, index) => rootSegment === pathSegments[index]); +}; + /** * Safe join path segments and resolve it * * In case resolved path are out of root path - thrown error */ export const joinPath = (root: string, ...segments: string[]) => { + const resolvedRoot = path.resolve(root); const resolvedPath = path.resolve(path.join(root, ...segments)); - if (!resolvedPath.startsWith(root)) { + if (!isRootedPath(resolvedRoot, resolvedPath)) { throw new TypeError('Resolved path is out of root directory'); } diff --git a/packages/app/src/electron/utils/ipc/index.ts b/packages/app/src/electron/utils/ipc/index.ts index 95750fa3e..1a0803f40 100644 --- a/packages/app/src/electron/utils/ipc/index.ts +++ b/packages/app/src/electron/utils/ipc/index.ts @@ -39,7 +39,9 @@ export const createChannel = (options: ChannelOptions) => { ): T => { // Return proxy object with virtual callbacks return new Proxy({} as any, { - get: function (_target, methodName: string) { + get: function (target, methodName: string) { + if (typeof methodName !== 'string') return target[methodName]; + const endpoint = getResolvedEndpointName(methodName); const mapper = mappers[methodName as keyof T]; return (...args: Parameters) => { @@ -59,8 +61,10 @@ export const createChannel = (options: ChannelOptions) => { callbacks: ApiToHandlers, ): (() => void) => { // Subscribe on channels - const cleanups = Object.entries(callbacks).map(([methodName, callback]) => - receiver(getResolvedEndpointName(methodName), callback), + const cleanups = Object.keys(callbacks).map((methodName) => + receiver(getResolvedEndpointName(methodName), (params) => + callbacks[methodName](params), + ), ); return () => { cleanups.forEach((cleanup) => cleanup());