diff --git a/cypress.config.ts b/cypress.config.ts index 5251686f..93f4f3f7 100644 --- a/cypress.config.ts +++ b/cypress.config.ts @@ -7,7 +7,7 @@ process.env.NODE_ENV = 'development' process.env.npm_package_name = 'nextcloud-e2e-test-server' -import type { RunExecOptions, RunExecResult } from './lib/docker.ts' +import type { RunExecOptions, RunExecResult } from './lib/docker/index.ts' import { defineConfig } from 'cypress' import vitePreprocessor from 'cypress-vite' @@ -20,7 +20,7 @@ import { startNextcloud, stopNextcloud, waitOnNextcloud, -} from './lib/docker.ts' +} from './lib/docker/index.ts' export default defineConfig({ projectId: 'h2z7r3', diff --git a/eslint.config.ts b/eslint.config.ts index fe469287..dee3db47 100644 --- a/eslint.config.ts +++ b/eslint.config.ts @@ -21,7 +21,7 @@ export default defineConfig([ { name: 'node-scripts', - files: ['lib/docker.ts', 'playwright/**/*.mjs'], + files: ['lib/docker/**/*.ts', 'playwright/**/*.mjs'], languageOptions: { globals: { ...globals.node, diff --git a/lib/commands/docker.ts b/lib/commands/docker.ts index e0b5008c..874f8835 100644 --- a/lib/commands/docker.ts +++ b/lib/commands/docker.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -import type { RunExecOptions, RunExecResult } from '../docker.ts' +import type { RunExecOptions, RunExecResult } from '../docker/index.ts' const defaultOptions = { failOnError: true, diff --git a/lib/commands/occ.ts b/lib/commands/occ.ts index 3bb09c35..6186e87e 100644 --- a/lib/commands/occ.ts +++ b/lib/commands/occ.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -import type { RunExecOptions, RunExecResult } from '../docker.ts' +import type { RunExecOptions, RunExecResult } from '../docker/index.ts' import { runCommand } from './docker.ts' diff --git a/lib/cypress.ts b/lib/cypress.ts index 3ad44b51..0da6e339 100644 --- a/lib/cypress.ts +++ b/lib/cypress.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -import type { RunExecOptions, RunExecResult } from './docker.ts' +import type { RunExecOptions, RunExecResult } from './docker/index.ts' import type { Selector } from './selectors/index.ts' import { getNc, restoreState, runCommand, runOccCommand, saveState } from './commands/index.ts' diff --git a/lib/docker.ts b/lib/docker.ts deleted file mode 100644 index c2a9405e..00000000 --- a/lib/docker.ts +++ /dev/null @@ -1,825 +0,0 @@ -/** - * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -import type { Container } from 'dockerode' -import type { Stream } from 'stream' -import type { Extract, Pack } from 'tar-stream' - -import Docker from 'dockerode' -import { XMLParser } from 'fast-xml-parser' -import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs' -import { basename, dirname, join, resolve, sep } from 'path' -import { PassThrough } from 'stream' -import { pipeline } from 'stream/promises' -import tarStreamer from 'tar-stream' -import waitOn from 'wait-on' -import { User } from './User.ts' - -const SERVER_IMAGE = 'ghcr.io/nextcloud/continuous-integration-shallow-server' - -/** Named volume mounted at `/var/www/html/apps-writable`, holds the mounted and cloned apps */ -const APPS_WRITABLE_VOLUME = 'apps_writable' - -// The server image ships PHP but no Composer, so it is downloaded on demand -const COMPOSER_VERSION = process.env.NEXTCLOUD_E2E_COMPOSER_VERSION || 'latest-stable' -const COMPOSER_PHAR = '/tmp/composer.phar' -/** `COMPOSER_HOME` used inside the server container, must be writable by `www-data` */ -const COMPOSER_HOME = '/tmp/composer-home' - -/** Path of the server log inside the container, it lives on the tmpfs mounted data directory */ -const NEXTCLOUD_LOG = '/var/www/html/data/nextcloud.log' - -export const docker = new Docker({ socketPath: process.env.DOCKER_SOCKET ?? '/var/run/docker.sock' }) - -// Store the container name, different names are used to prevent conflicts when testing multiple apps locally -let _containerName: string | null = null -// Store latest server branch used, will be used for vendored apps -let _serverBranch = 'master' - -/** - * Get the container name that is currently created and/or used by dockerode - */ -export function getContainerName(): string { - if (_containerName === null) { - const app = basename(process.cwd()).replace(' ', '') - _containerName = `nextcloud-e2e-test-server_${app}` - } - return _containerName -} - -/** - * Get the current container used - * Throws if not found - */ -export function getContainer(): Container { - return docker.getContainer(getContainerName()) -} - -interface StartOptions { - /** - * Force recreate the container even if an old one is found - * - * @default false - */ - forceRecreate?: boolean - - /** - * Additional mounts to create on the container - * You can pass a mapping from server path (relative to Nextcloud root) to your local file system - * - * @example ```js - * { config: '/path/to/local/config' } - * ``` - */ - mounts?: Record - - /** - * Optional port binding - * The default port (TCP 80) will be exposed to this host port - */ - exposePort?: number -} - -/** - * Start the testing container - * - * @param branch server branch to use (default 'master') - * @param mountApp bind mount app within server (`true` for autodetect, `false` to disable, or a string to force a path) (default true) - * @param options Optional parameters to configure the container creation - * @return Promise resolving to the IP address of the server - * @throws {Error} If Nextcloud container could not be started - */ -export async function startNextcloud(branch = 'master', mountApp: boolean | string = true, options: StartOptions = {}): Promise { - let appPath = mountApp === true ? process.cwd() : mountApp - let appId: string | undefined - let appVersion: string | undefined - if (appPath) { - console.log('Mounting app directories…') - while (appPath) { - const appInfoPath = resolve(join(appPath, 'appinfo', 'info.xml')) - if (existsSync(appInfoPath)) { - const parser = new XMLParser() - const xmlDoc = parser.parse(readFileSync(appInfoPath)) - appId = xmlDoc.info.id - appVersion = xmlDoc.info.version - console.log(`└─ Found ${appId} version ${appVersion}`) - break - } else { - // skip if root is reached or manual directory was set - if (appPath === sep || typeof mountApp === 'string') { - console.log('└─ No appinfo found') - appPath = false - break - } - appPath = join(appPath, '..') - } - } - } - - try { - await pullImage() - - // Getting latest image - console.log('\nChecking running containers… 🔍') - const localImage = await docker.listImages({ filters: `{"reference": ["${SERVER_IMAGE}"]}` }) - - // Remove old container if exists and not initialized by us - try { - const oldContainer = getContainer() - const oldContainerData = await oldContainer.inspect() - if (oldContainerData.State.Running) { - console.log('├─ Existing running container found') - if (options.forceRecreate === true) { - console.log('└─ Forced recreation of container was enabled, removing…') - } else if (localImage[0].Id !== oldContainerData.Image) { - console.log('└─ But running container is outdated, replacing…') - } else { - // Get container's IP - console.log('├─ Reusing that container') - const ip = await getContainerIP(oldContainer) - return ip - } - } else { - console.log('└─ None found!') - } - // Forcing any remnants to be removed just in case - await oldContainer.remove({ force: true }) - } catch { - console.log('└─ None found!') - } - - // Starting container - console.log('\nStarting Nextcloud container… 🚀') - console.log(`├─ Using branch '${branch}'`) - - // The volume outlives the container, so it has to be pruned to not carry - // apps cloned for a previous run (possibly of another branch) into the new container - await pruneAppsWritableVolume() - - const mounts: string[] = [] - Object.entries(options.mounts ?? {}) - .forEach(([server, local]) => mounts.push(`${local}:/var/www/html/${server}:ro`)) - - if (appPath !== false) { - mounts.push(`${appPath}:/var/www/html/apps-writable/${appId}:ro`) - } - - const PortBindings = !options.exposePort - ? undefined - : { - '80/tcp': [{ - HostIP: '0.0.0.0', - HostPort: options.exposePort.toString(), - }], - } - - // On macOS we need to expose the port since the docker container is running within a VM - const autoExposePort = process.platform === 'darwin' - - const container = await docker.createContainer({ - Image: SERVER_IMAGE, - name: getContainerName(), - Env: [`BRANCH=${branch}`, 'APCU=1'], - HostConfig: { - Binds: mounts.length > 0 ? mounts : undefined, - PortBindings, - PublishAllPorts: autoExposePort, - // Mount data directory in RAM for faster IO - Mounts: [{ - Target: '/var/www/html/data', - Source: '', - Type: 'tmpfs', - ReadOnly: false, - }, { - Target: '/var/www/html/apps-writable', - Source: APPS_WRITABLE_VOLUME, - Type: 'volume', - ReadOnly: false, - }], - }, - }) - await container.start() - - // Set proper permissions for the data folder - await runExec(['chown', '-R', 'www-data:www-data', '/var/www/html/data'], { container, user: 'root' }) - await runExec(['chmod', '0770', '/var/www/html/data'], { container, user: 'root' }) - - // Get container's IP - const ip = await getContainerIP(container) - console.log(`├─ Nextcloud container's IP is ${ip} 🌏`) - - _serverBranch = branch - - return ip - } catch (err) { - console.log('└─ Unable to start the container 🛑') - console.log(err) - stopNextcloud() - throw new Error('Unable to start the container', { cause: err }) - } -} - -/** - * Remove the `apps-writable` volume, so that a newly created container starts with an empty apps path - * - * Docker keeps the named volume around when the container is removed, meaning apps cloned by - * `configureNextcloud` would otherwise be reused - including apps of a different server branch. - */ -async function pruneAppsWritableVolume() { - try { - await docker.getVolume(APPS_WRITABLE_VOLUME).remove() - console.log('├─ Pruned the "apps-writable" volume') - } catch (error) { - // The volume does not exist (yet), nothing to prune - if ((error as { statusCode?: number }).statusCode === 404) { - return - } - throw new Error(`Unable to remove the "${APPS_WRITABLE_VOLUME}" volume`, { cause: error }) - } -} - -/** - * - */ -function pullImage() { - // Pulling images - console.log('\nPulling images… ⏳') - return new Promise((resolve, reject) => docker.pull(SERVER_IMAGE, (_: unknown, stream: Stream) => { - const onFinished = function(err: Error | null) { - if (!err) { - return resolve(true) - } - reject(err) - } - // https://github.com/apocas/dockerode/issues/357 - if (stream) { - docker.modem.followProgress(stream, onFinished) - } else { - reject('Failed to open stream') - } - })) - .then(() => console.log('└─ Done')) - .catch((err) => console.log(`└─ 🛑 FAILED! Trying to continue with existing image. (${err})`)) -} - -/** - * Configure Nextcloud - * - * Shipped apps that are missing from the server image are cloned and their Composer dependencies - * are installed. Set `NEXTCLOUD_E2E_COMPOSER_VERSION` to pin the Composer version used for that. - * - * @param apps List of default apps to install (default is ['viewer']) - * @param vendoredBranch The branch used for vendored apps, should match server (defaults to latest branch used for `startNextcloud` or fallsback to `master`) - * @param container Optional server container to use (defaults to current container) - */ -export async function configureNextcloud(apps = ['viewer'], vendoredBranch?: string, container?: Container) { - vendoredBranch = vendoredBranch || _serverBranch - - console.log('\nConfiguring Nextcloud…') - container = container ?? getContainer() - await runOcc('--version', { container, verbose: true }) - - // Be consistent for screenshots - await setSystemConfig('default_language', 'en', { container }) - await setSystemConfig('force_language', 'en', { container }) - await setSystemConfig('default_locale', 'en_US', { container }) - await setSystemConfig('force_locale', 'en_US', { container }) - await setSystemConfig('enforce_theme', 'light', { container }) - - // Checking apcu - console.log('├─ Checking APCu configuration... 👀') - const distributed = await getSystemConfig('memcache.distributed', { container }) - const local = await getSystemConfig('memcache.local', { container }) - const hashing = await getSystemConfig('hashing_default_password', { container }) - if (!distributed.includes('Memcache\\APCu') - || !local.includes('Memcache\\APCu') - || !hashing.includes('true')) { - console.log('└─ APCu is not properly configured 🛑') - throw new Error('APCu is not properly configured', { cause: { distributed, local, hashing } }) - } - console.log('│ └─ OK !') - - console.log('├─ Using "apps-writable" folder for mounted apps') - await runExec(['mkdir', '-p', '/var/www/html/apps-writable'], { container }) - await runExec(['chown', 'www-data:www-data', '/var/www/html/apps-writable'], { container, user: 'root' }) - const appsConfig = ` [ - [ - 'path' => '/var/www/html/apps', - 'url' => '/apps', - 'writable' => false, - ], - [ - 'path' => '/var/www/html/apps-writable', - 'url' => '/apps-writable', - 'writable' => true, - ], - ], -];` - const stream = tarStreamer.pack() - stream.entry({ name: 'apps.config.php' }, appsConfig) - stream.finalize() - await container.putArchive(asNodeStream(stream), { path: '/var/www/html/config' }) - - // Build app list, only now that "apps-writable" is a known apps path so that mounted apps show up - const { stdout: json } = await runOcc(['app:list', '--output', 'json'], { container }) - const applist = JSON.parse(json) - - // Enable apps and give status - for (const app of apps) { - if (app in applist.enabled) { - console.log(`├─ ${app} version ${applist.enabled[app]} already installed and enabled`) - } else if (app in applist.disabled) { - // built in or mounted already as the app under development - await runOcc(['app:enable', '--force', app], { container, verbose: true }) - } else { - const { stdout: jsonOutput } = await runExec(['cat', 'core/shipped.json'], { container }) - const { shippedApps } = JSON.parse(jsonOutput) - if (shippedApps.includes(app)) { - const branchOption = ['main', 'master'].includes(vendoredBranch) ? [] : [`--branch=${vendoredBranch}`] - await runExec( - ['git', 'clone', '--depth=1', ...branchOption, `https://github.com/nextcloud/${encodeURIComponent(app)}.git`, `apps-writable/${app}`], - { container, verbose: true }, - ) - await installComposerDependencies(app, container) - await runOcc(['app:enable', '--force', app], { container, verbose: true }) - } else { - // try appstore - await runOcc(['app:install', '--force', app], { container, verbose: true }) - } - } - } - console.log('└─ Nextcloud is now ready to use 🎉') -} - -/** - * Check whether a path exists inside the container - * - * @param path Absolute path to check - * @param container The server container to use - */ -async function pathExists(path: string, container: Container): Promise { - const { exitCode } = await runExec(['test', '-e', path], { container, failOnError: false }) - return exitCode === 0 -} - -/** - * Install the Composer dependencies of a cloned app - * - * A bare `git clone` is only usable as long as the app commits its dependencies. Since Nextcloud 34 - * `notifications` does not, and `OC_App::registerAutoloading()` then fatals on the missing - * `vendor/autoload.php`. Scripts are run on purpose, apps like that one only assemble the prefixed - * copies of their dependencies (`lib/Vendor`) in `post-install-cmd`. - * - * @param app The app id, cloned to `apps-writable/` - * @param container The server container to use - */ -async function installComposerDependencies(app: string, container: Container) { - const appPath = `/var/www/html/apps-writable/${app}` - if (!await pathExists(`${appPath}/composer.json`, container)) { - return - } - - await ensureComposer(container) - console.log(`│ ├─ Running 'composer install' for ${app}…`) - await runExec( - ['php', COMPOSER_PHAR, 'install', '--no-dev', '--no-interaction', '--no-progress', '--no-ansi'], - { container, workingDir: appPath, env: [`COMPOSER_HOME=${COMPOSER_HOME}`] }, - ) - console.log('│ └─ Done') -} - -/** - * Download the Composer binary into the container, unless it is already there - * - * @param container The server container to use - */ -async function ensureComposer(container: Container) { - if (await pathExists(COMPOSER_PHAR, container)) { - return - } - - console.log(`│ ├─ Downloading Composer ${COMPOSER_VERSION} into the container…`) - const url = `https://getcomposer.org/download/${COMPOSER_VERSION}/composer.phar` - await runExec(['curl', '--silent', '--show-error', '--location', '--fail', '--output', COMPOSER_PHAR, url], { container }) -} - -/** - * Setup test users - * - * @param container Optional server container to use (defaults to current container) - */ -export async function setupUsers(container?: Container) { - console.log('\nCreating test users… 👤') - const users = ['test1', 'test2', 'test3', 'test4', 'test5'] - .map((uid) => new User(uid)) - for (const user of users) { - await addUser(user, { container, verbose: true }) - } - console.log('└─ Done') -} - -/** - * Create a snapshot of the current database - * - * @param snapshot Name of the snapshot (default is a timestamp) - * @param container Optional server container to use (defaults to current container) - * @return Promise resolving to the snapshot name - */ -export async function createSnapshot(snapshot?: string, container?: Container): Promise { - const hash = new Date().toISOString().replace(/[^0-9]/g, '') - console.log('\nCreating init DB snapshot…') - await runExec(['cp', '/var/www/html/data/owncloud.db', `/var/www/html/data/owncloud.db-${snapshot ?? hash}`], { container, verbose: true }) - console.log('└─ Done') - return snapshot ?? hash -} - -/** - * Restore a snapshot of the database - * - * @param snapshot Name of the snapshot (default is 'init') - * @param container Optional server container to use (defaults to current container) - */ -export async function restoreSnapshot(snapshot = 'init', container?: Container) { - console.log('\nRestoring DB snapshot…') - await runExec(['cp', `/var/www/html/data/owncloud.db-${snapshot}`, '/var/www/html/data/owncloud.db'], { container, verbose: true }) - console.log('└─ Done') -} - -/** - * Read the server log (`data/nextcloud.log`) from the container. - * - * The data directory is a tmpfs and the container is removed after the run, - * so the log has to be fetched while the container still exists. - * - * @param container Optional server container to use (defaults to current container) - * @return The log contents, or an empty string if the server has not written a log - */ -export async function getNextcloudLog(container?: Container): Promise { - container = container ?? getContainer() - - let archive: NodeJS.ReadableStream - try { - archive = await container.getArchive({ path: NEXTCLOUD_LOG }) - } catch { - // No log written (yet), or the container is already gone - return '' - } - - // `getArchive` always answers with a tar stream, containing the single log entry - const extract = tarStreamer.extract() - const chunks: Buffer[] = [] - extract.on('entry', (_header, stream, next) => { - stream.on('data', (chunk) => chunks.push(chunk as Buffer)) - stream.on('end', () => next()) - }) - await pipeline(archive, asNodeStream(extract)) - - return Buffer.concat(chunks).toString('utf8') -} - -/** - * Save the server log (`data/nextcloud.log`) from the container to a local file. - * - * Must be called before {@link stopNextcloud}, the log is lost with the container. - * - * @param targetPath Local path to write the log to (default 'nextcloud.log' in the current directory) - * @param container Optional server container to use (defaults to current container) - * @return Whether a log was found and written - */ -export async function saveNextcloudLog(targetPath = 'nextcloud.log', container?: Container): Promise { - const log = await getNextcloudLog(container) - if (log === '') { - console.log('└─ No server log found in the container') - return false - } - - const target = resolve(targetPath) - mkdirSync(dirname(target), { recursive: true }) - writeFileSync(target, log) - console.log(`└─ Server log saved to ${target} 📝`) - return true -} - -interface StopOptions { - /** - * Local path to save the server log (`data/nextcloud.log`) to before the container is removed. - * - * @default process.env.NEXTCLOUD_E2E_LOG_FILE (disabled if unset) - */ - saveLogTo?: string -} - -/** - * Force stop the testing container - * - * @param options Optional parameters to configure the container removal - */ -export async function stopNextcloud(options: StopOptions = {}) { - try { - const container = getContainer() - - const logTarget = options.saveLogTo ?? process.env.NEXTCLOUD_E2E_LOG_FILE - if (logTarget) { - console.log('\nSaving Nextcloud server log…') - await saveNextcloudLog(logTarget, container) - } - - console.log('Stopping Nextcloud container…') - await container.remove({ force: true }) - console.log('└─ Nextcloud container removed 🥀') - } catch (err) { - console.log(err) - } -} - -/** - * Get the testing container's IP - * - * @param container name of the container - */ -export async function getContainerIP(container = getContainer()): Promise { - const containerInspect = await container.inspect() - const hostPort = containerInspect.NetworkSettings.Ports['80/tcp']?.[0]?.HostPort - - if (hostPort) { - return `localhost:${hostPort}` - } - - let ip = '' - let tries = 0 - while (ip === '' && tries < 10) { - tries++ - - try { - const containerInfo = await container.inspect() - const network = containerInfo.NetworkSettings.Networks.default - || containerInfo.NetworkSettings.Networks.bridge - || Object.values(containerInfo.NetworkSettings.Networks)[0] - if (network.IPAddress) { - ip = network.IPAddress - break - } - } catch { - // ignore and retry - } - - await sleep(1000 * tries) - } - - return ip -} - -/** - * Wait for Nextcloud to be ready - * - * @param ip - The IP address of the Nextcloud container - */ -export async function waitOnNextcloud(ip: string) { - console.log('├─ Waiting for Nextcloud to be ready… ⏳') - await waitOn({ resources: [`http://${ip}/index.php`] }) - console.log('└─ Done') -} - -export interface RunExecOptions { - /** - * The container to run the command in. If not provided, the current container will be used. - */ - container: Docker.Container - /** - * The user to run the command as. Defaults to 'www-data'. - */ - user: string - /** - * The command will throw an error if it exits with a non-zero exit code. Defaults to true. - */ - failOnError: boolean - /** - * Environment variables to set for the command. Defaults to an empty array. - */ - env: string[] - /** - * If true, the command's output will be printed to the console. Defaults to false. - */ - verbose: boolean - /** - * Working directory to run the command in. Defaults to the Nextcloud root. - */ - workingDir: string -} - -export type RunExecResult = { - stdout: string - stderr: string - exitCode: number -} - -/** - * Execute a command in the container and return stdout/stderr separately. - * - * @param command - The command to execute, either as a string or an array of strings (arguments) - * @param options - Options for executing the command - * @param options.container - The container to run the command in. If not provided, the current container will be used. - * @param options.user - The user to run the command as. Defaults to 'www-data'. - * @param options.verbose - If true, the command's output will be printed to the console. Defaults to false. - * @param options.env - Environment variables to set for the command. Defaults to an empty array. - * @param options.failOnError - The command will throw an error if it exits with a non-zero exit code. Defaults to true. - * @param options.workingDir - Working directory to run the command in. Defaults to the Nextcloud root. - */ -export async function runExec( - command: string | string[], - { container, user = 'www-data', verbose = false, env = [], failOnError = true, workingDir }: Partial = {}, -): Promise { - container = container || getContainer() - const exec = await container.exec({ - Cmd: typeof command === 'string' ? [command] : command, - AttachStdout: true, - AttachStderr: true, - User: user, - Env: env, - WorkingDir: workingDir, - }) - - return new Promise((resolve, reject) => { - const stdoutStream = new PassThrough() - const stderrStream = new PassThrough() - - const stdout: string[] = [] - const stderr: string[] = [] - - let settled = false - let finishedStreams = 0 - - const cleanup = () => { - stdoutStream.removeAllListeners() - stderrStream.removeAllListeners() - } - - const settleResolve = (result: RunExecResult) => { - if (settled) { - return - } - settled = true - cleanup() - resolve(result) - } - - const settleReject = (err: unknown) => { - if (settled) { - return - } - settled = true - cleanup() - reject(err) - } - - const maybeResolve = async () => { - finishedStreams++ - if (finishedStreams === 2) { - const inspectionResult = await exec.inspect() - const result = { - stdout: stdout.join(''), - stderr: stderr.join(''), - exitCode: inspectionResult.ExitCode ?? 0, - } - - if (result.exitCode && failOnError) { - settleReject(new Error('command exited with non-zero exit code', { cause: result })) - return - } - settleResolve(result) - } - } - - stdoutStream.on('data', (chunk) => { - const text = chunk.toString('utf8') - stdout.push(text) - if (verbose && text.trim()) { - console.log(`├─ stdout: ${text.trim().replace(/\n/gi, '\n├─ stdout: ')}`) - } - }) - - stderrStream.on('data', (chunk) => { - const text = chunk.toString('utf8') - stderr.push(text) - if (verbose && text.trim()) { - console.log(`├─ stderr: ${text.trim().replace(/\n/gi, '\n├─ stderr: ')}`) - } - }) - - stdoutStream.on('error', settleReject) - stderrStream.on('error', settleReject) - - stdoutStream.on('end', maybeResolve) - stderrStream.on('end', maybeResolve) - - exec.start({}, (err, stream) => { - if (err) { - settleReject(err) - return - } - if (!stream) { - settleReject(new Error('No exec stream returned')) - return - } - - stream.on('error', settleReject) - stream.on('end', () => { - stdoutStream.end() - stderrStream.end() - }) - - exec.modem.demuxStream(stream, stdoutStream, stderrStream) - }) - }) -} - -/** - * Execute an occ command in the container - * - * @param command - The occ command to execute, either as a string or an array of strings (arguments) - * @param options - Options for executing the command - * @param options.container - The container to run the command in. If not provided, the current container will be used. - * @param options.env - Environment variables to set for the command. Defaults to an empty array. - * @param options.verbose - If true, the command's output will be printed to the console. Defaults to false. - */ -export async function runOcc( - command: string | string[], - { container, env = [], verbose = false, ...rest }: Partial> = {}, -) { - const cmdArray = typeof command === 'string' ? [command] : command - return runExec(['php', 'occ', ...cmdArray], { ...rest, container, verbose, env }) -} - -/** - * Set a Nextcloud system config in the container. - * - * @param key - The config key to set - * @param value - The value to set for the config key - * @param options - Options for executing the command - * @param options.container - The container to run the command in. If not provided, the current container will be used. - */ -export function setSystemConfig(key: string, value: string, { container }: { container?: Docker.Container } = {}) { - return runOcc(['config:system:set', key, '--value', value], { container, verbose: true }) -} - -/** - * Get a Nextcloud system config value from the container. - * - * @param key - The config key to retrieve - * @param options - Options for executing the command - * @param options.container - The container to run the command in. If not provided, the current container will be used. - */ -export async function getSystemConfig( - key: string, - { container }: { container?: Docker.Container } = {}, -) { - const { stdout } = await runOcc(['config:system:get', key], { container }) - return stdout.trim() -} - -/** - * Add a user to the Nextcloud in the container. - * - * @param user - The user object containing userId and password - * @param options - Options for executing the command - * @param options.container - The container to run the command in. If not provided, the current container will be used. - * @param options.env - Environment variables to set for the command. Defaults to an empty array. - * @param options.verbose - If true, the command's output will be printed to the console. Defaults to false. - */ -export function addUser(user: User, { container, env = [], verbose = false }: Partial> = {}) { - return runOcc( - ['user:add', user.userId, '--password-from-env'], - { container, verbose, env: ['OC_PASS=' + user.password, ...env] }, - ) -} - -/** - * Present a `tar-stream` stream as the Node.js stream its consumers expect. - * - * `tar-stream` is typed as the `streamx` streams it is built on. Those behave - * like Node.js streams at runtime, but are not structurally assignable to them, so both - * `dockerode` and `stream.pipeline` need the stream to be cast. - * - * @param stream The pack or extract stream to cast - */ -function asNodeStream(stream: Pack): NodeJS.ReadableStream -function asNodeStream(stream: Extract): NodeJS.WritableStream -/** - * @param stream The pack or extract stream to cast - */ -function asNodeStream(stream: Pack | Extract) { - return stream as unknown as NodeJS.ReadableStream & NodeJS.WritableStream -} - -/** - * Pauses execution for a specified number of milliseconds. - * - * @param milliseconds - The number of milliseconds to sleep. - */ -function sleep(milliseconds: number) { - return new Promise((resolve) => setTimeout(resolve, milliseconds)) -} diff --git a/lib/docker/client.ts b/lib/docker/client.ts new file mode 100644 index 00000000..7edc651f --- /dev/null +++ b/lib/docker/client.ts @@ -0,0 +1,51 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import type { Container } from 'dockerode' + +import Docker from 'dockerode' +import { basename } from 'path' + +export const docker = new Docker({ socketPath: process.env.DOCKER_SOCKET ?? '/var/run/docker.sock' }) + +// Store the container name, different names are used to prevent conflicts when testing multiple apps locally +let _containerName: string | null = null +// Store latest server branch used, will be used for vendored apps +let _serverBranch = 'master' + +/** + * Get the container name that is currently created and/or used by dockerode + */ +export function getContainerName(): string { + if (_containerName === null) { + const app = basename(process.cwd()).replace(' ', '') + _containerName = `nextcloud-e2e-test-server_${app}` + } + return _containerName +} + +/** + * Get the current container used + * Throws if not found + */ +export function getContainer(): Container { + return docker.getContainer(getContainerName()) +} + +/** + * Get the server branch of the container started last, used as the default for vendored apps + */ +export function getServerBranch(): string { + return _serverBranch +} + +/** + * Remember the server branch a container was started with + * + * @param branch The branch passed to `startNextcloud` + */ +export function setServerBranch(branch: string): void { + _serverBranch = branch +} diff --git a/lib/docker/config.ts b/lib/docker/config.ts new file mode 100644 index 00000000..db9b7f75 --- /dev/null +++ b/lib/docker/config.ts @@ -0,0 +1,35 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import type Docker from 'dockerode' + +import { runOcc } from './exec.ts' + +/** + * Set a Nextcloud system config in the container. + * + * @param key - The config key to set + * @param value - The value to set for the config key + * @param options - Options for executing the command + * @param options.container - The container to run the command in. If not provided, the current container will be used. + */ +export function setSystemConfig(key: string, value: string, { container }: { container?: Docker.Container } = {}) { + return runOcc(['config:system:set', key, '--value', value], { container, verbose: true }) +} + +/** + * Get a Nextcloud system config value from the container. + * + * @param key - The config key to retrieve + * @param options - Options for executing the command + * @param options.container - The container to run the command in. If not provided, the current container will be used. + */ +export async function getSystemConfig( + key: string, + { container }: { container?: Docker.Container } = {}, +) { + const { stdout } = await runOcc(['config:system:get', key], { container }) + return stdout.trim() +} diff --git a/lib/docker/configure.ts b/lib/docker/configure.ts new file mode 100644 index 00000000..42404a37 --- /dev/null +++ b/lib/docker/configure.ts @@ -0,0 +1,150 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import type { Container } from 'dockerode' + +import tarStreamer from 'tar-stream' +import { getContainer, getServerBranch } from './client.ts' +import { getSystemConfig, setSystemConfig } from './config.ts' +import { pathExists, runExec, runOcc } from './exec.ts' +import { asNodeStream } from './internal.ts' + +// The server image ships PHP but no Composer, so it is downloaded on demand +const COMPOSER_VERSION = process.env.NEXTCLOUD_E2E_COMPOSER_VERSION || 'latest-stable' +const COMPOSER_PHAR = '/tmp/composer.phar' +/** `COMPOSER_HOME` used inside the server container, must be writable by `www-data` */ +const COMPOSER_HOME = '/tmp/composer-home' + +/** + * Configure Nextcloud + * + * Shipped apps that are missing from the server image are cloned and their Composer dependencies + * are installed. Set `NEXTCLOUD_E2E_COMPOSER_VERSION` to pin the Composer version used for that. + * + * @param apps List of default apps to install (default is ['viewer']) + * @param vendoredBranch The branch used for vendored apps, should match server (defaults to latest branch used for `startNextcloud` or fallsback to `master`) + * @param container Optional server container to use (defaults to current container) + */ +export async function configureNextcloud(apps = ['viewer'], vendoredBranch?: string, container?: Container) { + vendoredBranch = vendoredBranch || getServerBranch() + + console.log('\nConfiguring Nextcloud…') + container = container ?? getContainer() + await runOcc('--version', { container, verbose: true }) + + // Be consistent for screenshots + await setSystemConfig('default_language', 'en', { container }) + await setSystemConfig('force_language', 'en', { container }) + await setSystemConfig('default_locale', 'en_US', { container }) + await setSystemConfig('force_locale', 'en_US', { container }) + await setSystemConfig('enforce_theme', 'light', { container }) + + // Checking apcu + console.log('├─ Checking APCu configuration... 👀') + const distributed = await getSystemConfig('memcache.distributed', { container }) + const local = await getSystemConfig('memcache.local', { container }) + const hashing = await getSystemConfig('hashing_default_password', { container }) + if (!distributed.includes('Memcache\\APCu') + || !local.includes('Memcache\\APCu') + || !hashing.includes('true')) { + console.log('└─ APCu is not properly configured 🛑') + throw new Error('APCu is not properly configured', { cause: { distributed, local, hashing } }) + } + console.log('│ └─ OK !') + + console.log('├─ Using "apps-writable" folder for mounted apps') + await runExec(['mkdir', '-p', '/var/www/html/apps-writable'], { container }) + await runExec(['chown', 'www-data:www-data', '/var/www/html/apps-writable'], { container, user: 'root' }) + const appsConfig = ` [ + [ + 'path' => '/var/www/html/apps', + 'url' => '/apps', + 'writable' => false, + ], + [ + 'path' => '/var/www/html/apps-writable', + 'url' => '/apps-writable', + 'writable' => true, + ], + ], +];` + const stream = tarStreamer.pack() + stream.entry({ name: 'apps.config.php' }, appsConfig) + stream.finalize() + await container.putArchive(asNodeStream(stream), { path: '/var/www/html/config' }) + + // Build app list, only now that "apps-writable" is a known apps path so that mounted apps show up + const { stdout: json } = await runOcc(['app:list', '--output', 'json'], { container }) + const applist = JSON.parse(json) + + // Enable apps and give status + for (const app of apps) { + if (app in applist.enabled) { + console.log(`├─ ${app} version ${applist.enabled[app]} already installed and enabled`) + } else if (app in applist.disabled) { + // built in or mounted already as the app under development + await runOcc(['app:enable', '--force', app], { container, verbose: true }) + } else { + const { stdout: jsonOutput } = await runExec(['cat', 'core/shipped.json'], { container }) + const { shippedApps } = JSON.parse(jsonOutput) + if (shippedApps.includes(app)) { + const branchOption = ['main', 'master'].includes(vendoredBranch) ? [] : [`--branch=${vendoredBranch}`] + await runExec( + ['git', 'clone', '--depth=1', ...branchOption, `https://github.com/nextcloud/${encodeURIComponent(app)}.git`, `apps-writable/${app}`], + { container, verbose: true }, + ) + await installComposerDependencies(app, container) + await runOcc(['app:enable', '--force', app], { container, verbose: true }) + } else { + // try appstore + await runOcc(['app:install', '--force', app], { container, verbose: true }) + } + } + } + console.log('└─ Nextcloud is now ready to use 🎉') +} + +/** + * Install the Composer dependencies of a cloned app + * + * A bare `git clone` is only usable as long as the app commits its dependencies. Since Nextcloud 34 + * `notifications` does not, and `OC_App::registerAutoloading()` then fatals on the missing + * `vendor/autoload.php`. Scripts are run on purpose, apps like that one only assemble the prefixed + * copies of their dependencies (`lib/Vendor`) in `post-install-cmd`. + * + * @param app The app id, cloned to `apps-writable/` + * @param container The server container to use + */ +async function installComposerDependencies(app: string, container: Container) { + const appPath = `/var/www/html/apps-writable/${app}` + if (!await pathExists(`${appPath}/composer.json`, container)) { + return + } + + await ensureComposer(container) + console.log(`│ ├─ Running 'composer install' for ${app}…`) + await runExec( + ['php', COMPOSER_PHAR, 'install', '--no-dev', '--no-interaction', '--no-progress', '--no-ansi'], + { container, workingDir: appPath, env: [`COMPOSER_HOME=${COMPOSER_HOME}`] }, + ) + console.log('│ └─ Done') +} + +/** + * Download the Composer binary into the container, unless it is already there + * + * @param container The server container to use + */ +async function ensureComposer(container: Container) { + if (await pathExists(COMPOSER_PHAR, container)) { + return + } + + console.log(`│ ├─ Downloading Composer ${COMPOSER_VERSION} into the container…`) + const url = `https://getcomposer.org/download/${COMPOSER_VERSION}/composer.phar` + await runExec(['curl', '--silent', '--show-error', '--location', '--fail', '--output', COMPOSER_PHAR, url], { container }) +} diff --git a/lib/docker/exec.ts b/lib/docker/exec.ts new file mode 100644 index 00000000..aef16b0a --- /dev/null +++ b/lib/docker/exec.ts @@ -0,0 +1,191 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import type Docker from 'dockerode' +import type { Container } from 'dockerode' + +import { PassThrough } from 'stream' +import { getContainer } from './client.ts' + +export interface RunExecOptions { + /** + * The container to run the command in. If not provided, the current container will be used. + */ + container: Docker.Container + /** + * The user to run the command as. Defaults to 'www-data'. + */ + user: string + /** + * The command will throw an error if it exits with a non-zero exit code. Defaults to true. + */ + failOnError: boolean + /** + * Environment variables to set for the command. Defaults to an empty array. + */ + env: string[] + /** + * If true, the command's output will be printed to the console. Defaults to false. + */ + verbose: boolean + /** + * Working directory to run the command in. Defaults to the Nextcloud root. + */ + workingDir: string +} + +export type RunExecResult = { + stdout: string + stderr: string + exitCode: number +} + +/** + * Execute a command in the container and return stdout/stderr separately. + * + * @param command - The command to execute, either as a string or an array of strings (arguments) + * @param options - Options for executing the command + * @param options.container - The container to run the command in. If not provided, the current container will be used. + * @param options.user - The user to run the command as. Defaults to 'www-data'. + * @param options.verbose - If true, the command's output will be printed to the console. Defaults to false. + * @param options.env - Environment variables to set for the command. Defaults to an empty array. + * @param options.failOnError - The command will throw an error if it exits with a non-zero exit code. Defaults to true. + * @param options.workingDir - Working directory to run the command in. Defaults to the Nextcloud root. + */ +export async function runExec( + command: string | string[], + { container, user = 'www-data', verbose = false, env = [], failOnError = true, workingDir }: Partial = {}, +): Promise { + container = container || getContainer() + const exec = await container.exec({ + Cmd: typeof command === 'string' ? [command] : command, + AttachStdout: true, + AttachStderr: true, + User: user, + Env: env, + WorkingDir: workingDir, + }) + + return new Promise((resolve, reject) => { + const stdoutStream = new PassThrough() + const stderrStream = new PassThrough() + + const stdout: string[] = [] + const stderr: string[] = [] + + let settled = false + let finishedStreams = 0 + + const cleanup = () => { + stdoutStream.removeAllListeners() + stderrStream.removeAllListeners() + } + + const settleResolve = (result: RunExecResult) => { + if (settled) { + return + } + settled = true + cleanup() + resolve(result) + } + + const settleReject = (err: unknown) => { + if (settled) { + return + } + settled = true + cleanup() + reject(err) + } + + const maybeResolve = async () => { + finishedStreams++ + if (finishedStreams === 2) { + const inspectionResult = await exec.inspect() + const result = { + stdout: stdout.join(''), + stderr: stderr.join(''), + exitCode: inspectionResult.ExitCode ?? 0, + } + + if (result.exitCode && failOnError) { + settleReject(new Error('command exited with non-zero exit code', { cause: result })) + return + } + settleResolve(result) + } + } + + stdoutStream.on('data', (chunk) => { + const text = chunk.toString('utf8') + stdout.push(text) + if (verbose && text.trim()) { + console.log(`├─ stdout: ${text.trim().replace(/\n/gi, '\n├─ stdout: ')}`) + } + }) + + stderrStream.on('data', (chunk) => { + const text = chunk.toString('utf8') + stderr.push(text) + if (verbose && text.trim()) { + console.log(`├─ stderr: ${text.trim().replace(/\n/gi, '\n├─ stderr: ')}`) + } + }) + + stdoutStream.on('error', settleReject) + stderrStream.on('error', settleReject) + + stdoutStream.on('end', maybeResolve) + stderrStream.on('end', maybeResolve) + + exec.start({}, (err, stream) => { + if (err) { + settleReject(err) + return + } + if (!stream) { + settleReject(new Error('No exec stream returned')) + return + } + + stream.on('error', settleReject) + stream.on('end', () => { + stdoutStream.end() + stderrStream.end() + }) + + exec.modem.demuxStream(stream, stdoutStream, stderrStream) + }) + }) +} + +/** + * Execute an occ command in the container + * + * @param command - The occ command to execute, either as a string or an array of strings (arguments) + * @param options - Options for executing the command + * @param options.container - The container to run the command in. If not provided, the current container will be used. + * @param options.env - Environment variables to set for the command. Defaults to an empty array. + * @param options.verbose - If true, the command's output will be printed to the console. Defaults to false. + */ +export async function runOcc( + command: string | string[], + { container, env = [], verbose = false, ...rest }: Partial> = {}, +) { + const cmdArray = typeof command === 'string' ? [command] : command + return runExec(['php', 'occ', ...cmdArray], { ...rest, container, verbose, env }) +} + +/** + * Check whether a path exists inside the container + * + * @param path Absolute path to check + * @param container The server container to use + */ +export async function pathExists(path: string, container: Container): Promise { + const { exitCode } = await runExec(['test', '-e', path], { container, failOnError: false }) + return exitCode === 0 +} diff --git a/lib/docker/index.ts b/lib/docker/index.ts new file mode 100644 index 00000000..301f5368 --- /dev/null +++ b/lib/docker/index.ts @@ -0,0 +1,15 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +export type { RunExecOptions, RunExecResult } from './exec.ts' + +export { docker, getContainer, getContainerName } from './client.ts' +export { getSystemConfig, setSystemConfig } from './config.ts' +export { configureNextcloud } from './configure.ts' +export { runExec, runOcc } from './exec.ts' +export { getContainerIP, startNextcloud, stopNextcloud, waitOnNextcloud } from './lifecycle.ts' +export { getNextcloudLog, saveNextcloudLog } from './logs.ts' +export { createSnapshot, restoreSnapshot } from './snapshots.ts' +export { addUser, setupUsers } from './users.ts' diff --git a/lib/docker/internal.ts b/lib/docker/internal.ts new file mode 100644 index 00000000..41c42105 --- /dev/null +++ b/lib/docker/internal.ts @@ -0,0 +1,33 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import type { Extract, Pack } from 'tar-stream' + +/** + * Present a `tar-stream` stream as the Node.js stream its consumers expect. + * + * `tar-stream` is typed as the `streamx` streams it is built on. Those behave + * like Node.js streams at runtime, but are not structurally assignable to them, so both + * `dockerode` and `stream.pipeline` need the stream to be cast. + * + * @param stream The pack or extract stream to cast + */ +export function asNodeStream(stream: Pack): NodeJS.ReadableStream +export function asNodeStream(stream: Extract): NodeJS.WritableStream +/** + * @param stream The pack or extract stream to cast + */ +export function asNodeStream(stream: Pack | Extract) { + return stream as unknown as NodeJS.ReadableStream & NodeJS.WritableStream +} + +/** + * Pauses execution for a specified number of milliseconds. + * + * @param milliseconds - The number of milliseconds to sleep. + */ +export function sleep(milliseconds: number) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)) +} diff --git a/lib/docker/lifecycle.ts b/lib/docker/lifecycle.ts new file mode 100644 index 00000000..93fa4319 --- /dev/null +++ b/lib/docker/lifecycle.ts @@ -0,0 +1,287 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import type { Container } from 'dockerode' +import type { Stream } from 'stream' + +import { XMLParser } from 'fast-xml-parser' +import { existsSync, readFileSync } from 'fs' +import { join, resolve, sep } from 'path' +import waitOn from 'wait-on' +import { docker, getContainer, getContainerName, setServerBranch } from './client.ts' +import { runExec } from './exec.ts' +import { sleep } from './internal.ts' +import { saveNextcloudLog } from './logs.ts' +import { APPS_WRITABLE_VOLUME, pruneAppsWritableVolume } from './volumes.ts' + +const SERVER_IMAGE = 'ghcr.io/nextcloud/continuous-integration-shallow-server' + +interface StartOptions { + /** + * Force recreate the container even if an old one is found + * + * @default false + */ + forceRecreate?: boolean + + /** + * Additional mounts to create on the container + * You can pass a mapping from server path (relative to Nextcloud root) to your local file system + * + * @example ```js + * { config: '/path/to/local/config' } + * ``` + */ + mounts?: Record + + /** + * Optional port binding + * The default port (TCP 80) will be exposed to this host port + */ + exposePort?: number +} + +/** + * Start the testing container + * + * @param branch server branch to use (default 'master') + * @param mountApp bind mount app within server (`true` for autodetect, `false` to disable, or a string to force a path) (default true) + * @param options Optional parameters to configure the container creation + * @return Promise resolving to the IP address of the server + * @throws {Error} If Nextcloud container could not be started + */ +export async function startNextcloud(branch = 'master', mountApp: boolean | string = true, options: StartOptions = {}): Promise { + let appPath = mountApp === true ? process.cwd() : mountApp + let appId: string | undefined + let appVersion: string | undefined + if (appPath) { + console.log('Mounting app directories…') + while (appPath) { + const appInfoPath = resolve(join(appPath, 'appinfo', 'info.xml')) + if (existsSync(appInfoPath)) { + const parser = new XMLParser() + const xmlDoc = parser.parse(readFileSync(appInfoPath)) + appId = xmlDoc.info.id + appVersion = xmlDoc.info.version + console.log(`└─ Found ${appId} version ${appVersion}`) + break + } else { + // skip if root is reached or manual directory was set + if (appPath === sep || typeof mountApp === 'string') { + console.log('└─ No appinfo found') + appPath = false + break + } + appPath = join(appPath, '..') + } + } + } + + try { + await pullImage() + + // Getting latest image + console.log('\nChecking running containers… 🔍') + const localImage = await docker.listImages({ filters: `{"reference": ["${SERVER_IMAGE}"]}` }) + + // Remove old container if exists and not initialized by us + try { + const oldContainer = getContainer() + const oldContainerData = await oldContainer.inspect() + if (oldContainerData.State.Running) { + console.log('├─ Existing running container found') + if (options.forceRecreate === true) { + console.log('└─ Forced recreation of container was enabled, removing…') + } else if (localImage[0].Id !== oldContainerData.Image) { + console.log('└─ But running container is outdated, replacing…') + } else { + // Get container's IP + console.log('├─ Reusing that container') + const ip = await getContainerIP(oldContainer) + return ip + } + } else { + console.log('└─ None found!') + } + // Forcing any remnants to be removed just in case + await oldContainer.remove({ force: true }) + } catch { + console.log('└─ None found!') + } + + // Starting container + console.log('\nStarting Nextcloud container… 🚀') + console.log(`├─ Using branch '${branch}'`) + + // The volume outlives the container, so it has to be pruned to not carry + // apps cloned for a previous run (possibly of another branch) into the new container + await pruneAppsWritableVolume() + + const mounts: string[] = [] + Object.entries(options.mounts ?? {}) + .forEach(([server, local]) => mounts.push(`${local}:/var/www/html/${server}:ro`)) + + if (appPath !== false) { + mounts.push(`${appPath}:/var/www/html/apps-writable/${appId}:ro`) + } + + const PortBindings = !options.exposePort + ? undefined + : { + '80/tcp': [{ + HostIP: '0.0.0.0', + HostPort: options.exposePort.toString(), + }], + } + + // On macOS we need to expose the port since the docker container is running within a VM + const autoExposePort = process.platform === 'darwin' + + const container = await docker.createContainer({ + Image: SERVER_IMAGE, + name: getContainerName(), + Env: [`BRANCH=${branch}`, 'APCU=1'], + HostConfig: { + Binds: mounts.length > 0 ? mounts : undefined, + PortBindings, + PublishAllPorts: autoExposePort, + // Mount data directory in RAM for faster IO + Mounts: [{ + Target: '/var/www/html/data', + Source: '', + Type: 'tmpfs', + ReadOnly: false, + }, { + Target: '/var/www/html/apps-writable', + Source: APPS_WRITABLE_VOLUME, + Type: 'volume', + ReadOnly: false, + }], + }, + }) + await container.start() + + // Set proper permissions for the data folder + await runExec(['chown', '-R', 'www-data:www-data', '/var/www/html/data'], { container, user: 'root' }) + await runExec(['chmod', '0770', '/var/www/html/data'], { container, user: 'root' }) + + // Get container's IP + const ip = await getContainerIP(container) + console.log(`├─ Nextcloud container's IP is ${ip} 🌏`) + + setServerBranch(branch) + + return ip + } catch (err) { + console.log('└─ Unable to start the container 🛑') + console.log(err) + stopNextcloud() + throw new Error('Unable to start the container', { cause: err }) + } +} + +/** + * + */ +function pullImage() { + // Pulling images + console.log('\nPulling images… ⏳') + return new Promise((resolve, reject) => docker.pull(SERVER_IMAGE, (_: unknown, stream: Stream) => { + const onFinished = function(err: Error | null) { + if (!err) { + return resolve(true) + } + reject(err) + } + // https://github.com/apocas/dockerode/issues/357 + if (stream) { + docker.modem.followProgress(stream, onFinished) + } else { + reject('Failed to open stream') + } + })) + .then(() => console.log('└─ Done')) + .catch((err) => console.log(`└─ 🛑 FAILED! Trying to continue with existing image. (${err})`)) +} + +interface StopOptions { + /** + * Local path to save the server log (`data/nextcloud.log`) to before the container is removed. + * + * @default process.env.NEXTCLOUD_E2E_LOG_FILE (disabled if unset) + */ + saveLogTo?: string +} + +/** + * Force stop the testing container + * + * @param options Optional parameters to configure the container removal + */ +export async function stopNextcloud(options: StopOptions = {}) { + try { + const container = getContainer() + + const logTarget = options.saveLogTo ?? process.env.NEXTCLOUD_E2E_LOG_FILE + if (logTarget) { + console.log('\nSaving Nextcloud server log…') + await saveNextcloudLog(logTarget, container) + } + + console.log('Stopping Nextcloud container…') + await container.remove({ force: true }) + console.log('└─ Nextcloud container removed 🥀') + } catch (err) { + console.log(err) + } +} + +/** + * Get the testing container's IP + * + * @param container name of the container + */ +export async function getContainerIP(container: Container = getContainer()): Promise { + const containerInspect = await container.inspect() + const hostPort = containerInspect.NetworkSettings.Ports['80/tcp']?.[0]?.HostPort + + if (hostPort) { + return `localhost:${hostPort}` + } + + let ip = '' + let tries = 0 + while (ip === '' && tries < 10) { + tries++ + + try { + const containerInfo = await container.inspect() + const network = containerInfo.NetworkSettings.Networks.default + || containerInfo.NetworkSettings.Networks.bridge + || Object.values(containerInfo.NetworkSettings.Networks)[0] + if (network.IPAddress) { + ip = network.IPAddress + break + } + } catch { + // ignore and retry + } + + await sleep(1000 * tries) + } + + return ip +} + +/** + * Wait for Nextcloud to be ready + * + * @param ip - The IP address of the Nextcloud container + */ +export async function waitOnNextcloud(ip: string) { + console.log('├─ Waiting for Nextcloud to be ready… ⏳') + await waitOn({ resources: [`http://${ip}/index.php`] }) + console.log('└─ Done') +} diff --git a/lib/docker/logs.ts b/lib/docker/logs.ts new file mode 100644 index 00000000..eeef4a8c --- /dev/null +++ b/lib/docker/logs.ts @@ -0,0 +1,71 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import type { Container } from 'dockerode' + +import { mkdirSync, writeFileSync } from 'fs' +import { dirname, resolve } from 'path' +import { pipeline } from 'stream/promises' +import tarStreamer from 'tar-stream' +import { getContainer } from './client.ts' +import { asNodeStream } from './internal.ts' + +/** Path of the server log inside the container, it lives on the tmpfs mounted data directory */ +const NEXTCLOUD_LOG = '/var/www/html/data/nextcloud.log' + +/** + * Read the server log (`data/nextcloud.log`) from the container. + * + * The data directory is a tmpfs and the container is removed after the run, + * so the log has to be fetched while the container still exists. + * + * @param container Optional server container to use (defaults to current container) + * @return The log contents, or an empty string if the server has not written a log + */ +export async function getNextcloudLog(container?: Container): Promise { + container = container ?? getContainer() + + let archive: NodeJS.ReadableStream + try { + archive = await container.getArchive({ path: NEXTCLOUD_LOG }) + } catch { + // No log written (yet), or the container is already gone + return '' + } + + // `getArchive` always answers with a tar stream, containing the single log entry + const extract = tarStreamer.extract() + const chunks: Buffer[] = [] + extract.on('entry', (_header, stream, next) => { + stream.on('data', (chunk) => chunks.push(chunk as Buffer)) + stream.on('end', () => next()) + }) + await pipeline(archive, asNodeStream(extract)) + + return Buffer.concat(chunks).toString('utf8') +} + +/** + * Save the server log (`data/nextcloud.log`) from the container to a local file. + * + * Must be called before {@link stopNextcloud}, the log is lost with the container. + * + * @param targetPath Local path to write the log to (default 'nextcloud.log' in the current directory) + * @param container Optional server container to use (defaults to current container) + * @return Whether a log was found and written + */ +export async function saveNextcloudLog(targetPath = 'nextcloud.log', container?: Container): Promise { + const log = await getNextcloudLog(container) + if (log === '') { + console.log('└─ No server log found in the container') + return false + } + + const target = resolve(targetPath) + mkdirSync(dirname(target), { recursive: true }) + writeFileSync(target, log) + console.log(`└─ Server log saved to ${target} 📝`) + return true +} diff --git a/lib/docker/snapshots.ts b/lib/docker/snapshots.ts new file mode 100644 index 00000000..6cb7d07d --- /dev/null +++ b/lib/docker/snapshots.ts @@ -0,0 +1,35 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import type { Container } from 'dockerode' + +import { runExec } from './exec.ts' + +/** + * Create a snapshot of the current database + * + * @param snapshot Name of the snapshot (default is a timestamp) + * @param container Optional server container to use (defaults to current container) + * @return Promise resolving to the snapshot name + */ +export async function createSnapshot(snapshot?: string, container?: Container): Promise { + const hash = new Date().toISOString().replace(/[^0-9]/g, '') + console.log('\nCreating init DB snapshot…') + await runExec(['cp', '/var/www/html/data/owncloud.db', `/var/www/html/data/owncloud.db-${snapshot ?? hash}`], { container, verbose: true }) + console.log('└─ Done') + return snapshot ?? hash +} + +/** + * Restore a snapshot of the database + * + * @param snapshot Name of the snapshot (default is 'init') + * @param container Optional server container to use (defaults to current container) + */ +export async function restoreSnapshot(snapshot = 'init', container?: Container) { + console.log('\nRestoring DB snapshot…') + await runExec(['cp', `/var/www/html/data/owncloud.db-${snapshot}`, '/var/www/html/data/owncloud.db'], { container, verbose: true }) + console.log('└─ Done') +} diff --git a/lib/docker/users.ts b/lib/docker/users.ts new file mode 100644 index 00000000..3ebe8151 --- /dev/null +++ b/lib/docker/users.ts @@ -0,0 +1,41 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import type { Container } from 'dockerode' +import type { RunExecOptions } from './exec.ts' + +import { User } from '../User.ts' +import { runOcc } from './exec.ts' + +/** + * Add a user to the Nextcloud in the container. + * + * @param user - The user object containing userId and password + * @param options - Options for executing the command + * @param options.container - The container to run the command in. If not provided, the current container will be used. + * @param options.env - Environment variables to set for the command. Defaults to an empty array. + * @param options.verbose - If true, the command's output will be printed to the console. Defaults to false. + */ +export function addUser(user: User, { container, env = [], verbose = false }: Partial> = {}) { + return runOcc( + ['user:add', user.userId, '--password-from-env'], + { container, verbose, env: ['OC_PASS=' + user.password, ...env] }, + ) +} + +/** + * Setup test users + * + * @param container Optional server container to use (defaults to current container) + */ +export async function setupUsers(container?: Container) { + console.log('\nCreating test users… 👤') + const users = ['test1', 'test2', 'test3', 'test4', 'test5'] + .map((uid) => new User(uid)) + for (const user of users) { + await addUser(user, { container, verbose: true }) + } + console.log('└─ Done') +} diff --git a/lib/docker/volumes.ts b/lib/docker/volumes.ts new file mode 100644 index 00000000..7357dcb3 --- /dev/null +++ b/lib/docker/volumes.ts @@ -0,0 +1,28 @@ +/** + * SPDX-FileCopyrightText: 2022 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +import { docker } from './client.ts' + +/** Named volume mounted at `/var/www/html/apps-writable`, holds the mounted and cloned apps */ +export const APPS_WRITABLE_VOLUME = 'apps_writable' + +/** + * Remove the `apps-writable` volume, so that a newly created container starts with an empty apps path + * + * Docker keeps the named volume around when the container is removed, meaning apps cloned by + * `configureNextcloud` would otherwise be reused - including apps of a different server branch. + */ +export async function pruneAppsWritableVolume() { + try { + await docker.getVolume(APPS_WRITABLE_VOLUME).remove() + console.log('├─ Pruned the "apps-writable" volume') + } catch (error) { + // The volume does not exist (yet), nothing to prune + if ((error as { statusCode?: number }).statusCode === 404) { + return + } + throw new Error(`Unable to remove the "${APPS_WRITABLE_VOLUME}" volume`, { cause: error }) + } +} diff --git a/lib/index.ts b/lib/index.ts index d5c96867..0603fb17 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -3,5 +3,5 @@ * SPDX-License-Identifier: AGPL-3.0-or-later */ -export * from './docker.ts' +export * from './docker/index.ts' export * from './User.ts' diff --git a/lib/playwright.ts b/lib/playwright.ts index ba099532..e315c3ab 100644 --- a/lib/playwright.ts +++ b/lib/playwright.ts @@ -5,7 +5,7 @@ import type { APIRequestContext } from 'playwright' -import { addUser } from './docker.ts' +import { addUser } from './docker/index.ts' import { User } from './User.ts' /** diff --git a/package.json b/package.json index aed53cb2..e4e445a9 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "require": "./dist/selectors.cjs" }, "./docker": { - "types": "./dist/docker.d.ts", + "types": "./dist/docker/index.d.ts", "import": "./dist/docker.mjs", "require": "./dist/docker.cjs" }, diff --git a/tests/docker.spec.ts b/tests/docker.spec.ts index e4588c39..ccf04dc8 100644 --- a/tests/docker.spec.ts +++ b/tests/docker.spec.ts @@ -5,7 +5,7 @@ import * as expect from 'node:assert' import { after, before, describe, test } from 'node:test' -import { configureNextcloud, docker, getContainer, runExec, runOcc, startNextcloud, stopNextcloud, waitOnNextcloud } from '../lib/docker.ts' +import { configureNextcloud, docker, getContainer, runExec, runOcc, startNextcloud, stopNextcloud, waitOnNextcloud } from '../lib/docker/index.ts' describe('Docker: Pre-installation of apps', async () => { before(async () => { diff --git a/tests/runExec.spec.ts b/tests/runExec.spec.ts index 3a441df4..d43fd2eb 100644 --- a/tests/runExec.spec.ts +++ b/tests/runExec.spec.ts @@ -7,7 +7,7 @@ import type { Container } from 'dockerode' import assert from 'node:assert/strict' import { after, before, describe, test } from 'node:test' -import { docker, getContainer, runExec, startNextcloud, stopNextcloud, waitOnNextcloud } from '../lib/docker.ts' +import { docker, getContainer, runExec, startNextcloud, stopNextcloud, waitOnNextcloud } from '../lib/docker/index.ts' describe('Docker: runExec', async () => { let container: Container diff --git a/vite.config.mts b/vite.config.mts index a92332ff..e79633f1 100644 --- a/vite.config.mts +++ b/vite.config.mts @@ -10,7 +10,7 @@ export default createLibConfig({ index: join(__dirname, 'lib/index.ts'), commands: join(__dirname, 'lib/commands/index.ts'), selectors: join(__dirname, 'lib/selectors/index.ts'), - docker: join(__dirname, 'lib/docker.ts'), + docker: join(__dirname, 'lib/docker/index.ts'), cypress: join(__dirname, 'lib/cypress.ts'), playwright: join(__dirname, 'lib/playwright.ts'), }, {