diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml index a3c130a..d899567 100644 --- a/.github/workflows/build-and-release.yml +++ b/.github/workflows/build-and-release.yml @@ -4,9 +4,32 @@ on: push: tags: - v*.*.* + workflow_dispatch: + inputs: + server_image_tag: + description: Server image tag to package. Defaults to the release tag. + required: false + server_image_digest: + description: Optional server image digest. Overrides the tag when set. + required: false + neuroglancer_plugin_tag: + description: Neuroglancer plugin release tag. Defaults to the release tag. + required: false + neuroglancer_plugin_artifact: + description: Neuroglancer plugin artifact name. Defaults from the plugin tag. + required: false + autoseg_plugin_tag: + description: Automatic segmentation plugin release tag. Defaults to the release tag. + required: false + autoseg_cpu_plugin_artifact: + description: CPU automatic segmentation plugin artifact name. Defaults from the plugin tag. + required: false + autoseg_cuda_plugin_artifact: + description: CUDA automatic segmentation plugin artifact name. Defaults from the plugin tag. + required: false env: - GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GH_TOKEN: "${{ secrets.OUROBOROS_RELEASE_ASSET_TOKEN || secrets.GITHUB_TOKEN }}" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" jobs: @@ -19,6 +42,10 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] + package: + - flavor: core + - flavor: plugins-cpu + - flavor: plugins-cuda steps: - name: Check out Git repository @@ -35,7 +62,18 @@ jobs: - name: Prepare registry-backed server resources run: npm run prepare:production-server env: - OUROBOROS_SERVER_IMAGE_TAG: ${{ github.ref_name }} + OUROBOROS_SERVER_IMAGE_TAG: ${{ github.event.inputs.server_image_tag || github.ref_name }} + OUROBOROS_SERVER_IMAGE_DIGEST: ${{ github.event.inputs.server_image_digest }} + + - name: Prepare production package flavor + run: npm run prepare:package-flavor + env: + OUROBOROS_PACKAGE_FLAVOR: ${{ matrix.package.flavor }} + OUROBOROS_NEUROGLANCER_PLUGIN_TAG: ${{ github.event.inputs.neuroglancer_plugin_tag || github.ref_name }} + OUROBOROS_NEUROGLANCER_PLUGIN_ARTIFACT: ${{ github.event.inputs.neuroglancer_plugin_artifact }} + OUROBOROS_AUTOSEG_PLUGIN_TAG: ${{ github.event.inputs.autoseg_plugin_tag || github.ref_name }} + OUROBOROS_AUTOSEG_CPU_PLUGIN_ARTIFACT: ${{ github.event.inputs.autoseg_cpu_plugin_artifact }} + OUROBOROS_AUTOSEG_CUDA_PLUGIN_ARTIFACT: ${{ github.event.inputs.autoseg_cuda_plugin_artifact }} - name: build-linux if: matrix.os == 'ubuntu-latest' @@ -49,6 +87,11 @@ jobs: if: matrix.os == 'windows-latest' run: npm run build:win + - name: Add package flavor to artifact names + run: npm run rename:release-artifacts + env: + OUROBOROS_PACKAGE_FLAVOR: ${{ matrix.package.flavor }} + - name: release uses: softprops/action-gh-release@v1 with: diff --git a/.gitignore b/.gitignore index c2bbd98..77a2b15 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ out *.log* documentation/site extra-resources +.package-plugin-artifacts data ignoreme.py temp diff --git a/documentation/docs/development/production-package-flavors.md b/documentation/docs/development/production-package-flavors.md new file mode 100644 index 0000000..4480768 --- /dev/null +++ b/documentation/docs/development/production-package-flavors.md @@ -0,0 +1,52 @@ +# Production package flavors + +The release workflow builds three production package flavors for each supported +desktop runner: + +- `core`: Ouroboros only, with the production server compose file pointing at + the registry-published server image. +- `plugins-cpu`: the core package plus bundled Neuroglancer and Automatic + Segmentation plugin artifacts. The automatic segmentation plugin uses its CPU + backend image. +- `plugins-cuda`: the core package plus bundled Neuroglancer and Automatic + Segmentation plugin artifacts. The automatic segmentation plugin uses its + CUDA backend image and Docker Compose GPU reservation. + +Plugin flavors stage release artifacts into +`extra-resources/preinstalled-plugins` before Electron Builder runs. The app +installs those bundled plugin folders into the user plugin directory on startup, +so production packages can ship with plugins already available. + +## Release inputs + +Tag releases default every image and plugin artifact to the Ouroboros release +tag. Manual workflow runs can override these inputs: + +- `server_image_tag` or `server_image_digest` +- `neuroglancer_plugin_tag` +- `neuroglancer_plugin_artifact` +- `autoseg_plugin_tag` +- `autoseg_cpu_plugin_artifact` +- `autoseg_cuda_plugin_artifact` + +The default plugin artifact names are: + +- `neuroglancer-plugin-.zip` +- `auto-segmentation--cpu.zip` +- `auto-segmentation--cuda.zip` + +If plugin release repositories are private, set +`OUROBOROS_RELEASE_ASSET_TOKEN` to a token that can read those release assets. +The workflow falls back to `GITHUB_TOKEN` when that secret is not set. + +## Local checks + +Prepare the core package resources with: + +```sh +npm run prepare:production-server +OUROBOROS_PACKAGE_FLAVOR=core npm run prepare:package-flavor +``` + +Plugin flavors can be checked against local release zips by placing them in +`.package-plugin-artifacts` or setting `OUROBOROS_PLUGIN_ARTIFACT_DIR`. diff --git a/documentation/mkdocs.yml b/documentation/mkdocs.yml index f0c7854..62a5a62 100644 --- a/documentation/mkdocs.yml +++ b/documentation/mkdocs.yml @@ -17,6 +17,7 @@ nav: - Plugins: https://github.com/ChengLabResearch/ouroboros/blob/main/plugins/plugin-template/README.md - Docker Builds: 'development/docker-builds.md' - Registry Server Images: 'development/registry-server-images.md' + - Production Package Flavors: 'development/production-package-flavors.md' - Releases: 'https://github.com/ChengLabResearch/ouroboros/releases' - Repository: https://github.com/ChengLabResearch/ouroboros diff --git a/package.json b/package.json index 20a3849..33fed92 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,8 @@ "dev": "electron-vite dev", "build": "npm run typecheck && electron-vite build", "prepare:production-server": "node scripts/prepare-production-server.mjs", + "prepare:package-flavor": "node scripts/prepare-package-flavor.mjs", + "rename:release-artifacts": "node scripts/rename-release-artifacts.mjs", "postinstall": "electron-builder install-app-deps", "build:unpack": "npm run build && electron-builder --dir", "build:win": "electron-vite build && electron-builder --win --config --publish never", diff --git a/scripts/prepare-package-flavor.mjs b/scripts/prepare-package-flavor.mjs new file mode 100644 index 0000000..f07913c --- /dev/null +++ b/scripts/prepare-package-flavor.mjs @@ -0,0 +1,217 @@ +import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises' +import { existsSync } from 'node:fs' +import { isAbsolute, join } from 'node:path' +import { spawn } from 'node:child_process' + +const supportedFlavors = new Set(['core', 'plugins-cpu', 'plugins-cuda']) + +const root = process.cwd() +const packageJson = JSON.parse(await readFile(join(root, 'package.json'), 'utf8')) +const flavor = process.env.OUROBOROS_PACKAGE_FLAVOR ?? 'core' + +if (!supportedFlavors.has(flavor)) { + throw new Error( + `Unsupported OUROBOROS_PACKAGE_FLAVOR "${flavor}". Expected one of: ${[ + ...supportedFlavors + ].join(', ')}` + ) +} + +const extraResourcesDir = join(root, 'extra-resources') +const preinstalledPluginDir = join(extraResourcesDir, 'preinstalled-plugins') +const artifactDir = resolvePathFromRoot( + process.env.OUROBOROS_PLUGIN_ARTIFACT_DIR ?? '.package-plugin-artifacts' +) +const releaseTag = process.env.GITHUB_REF_NAME ?? `v${packageJson.version}` +const plugins = [] + +await mkdir(extraResourcesDir, { recursive: true }) +await rm(preinstalledPluginDir, { recursive: true, force: true }) + +if (flavor !== 'core') { + await mkdir(preinstalledPluginDir, { recursive: true }) + + const neuroglancerTag = process.env.OUROBOROS_NEUROGLANCER_PLUGIN_TAG || releaseTag + await installPlugin({ + id: 'neuroglancer-plugin', + label: 'Neuroglancer', + repo: 'ChengLabResearch/neuroglancer-plugin', + tag: neuroglancerTag, + artifact: + process.env.OUROBOROS_NEUROGLANCER_PLUGIN_ARTIFACT || + `neuroglancer-plugin-${neuroglancerTag}.zip` + }) + + const autosegTag = process.env.OUROBOROS_AUTOSEG_PLUGIN_TAG || releaseTag + const autosegVariant = flavor === 'plugins-cuda' ? 'cuda' : 'cpu' + await installPlugin({ + id: 'auto-segmentation', + label: `Automatic Segmentation (${autosegVariant.toUpperCase()})`, + repo: 'ChengLabResearch/ouroboros_autoseg_plugin', + tag: autosegTag, + artifact: + process.env[`OUROBOROS_AUTOSEG_${autosegVariant.toUpperCase()}_PLUGIN_ARTIFACT`] || + `auto-segmentation-${autosegTag}-${autosegVariant}.zip`, + variant: autosegVariant + }) +} + +await writeFile( + join(extraResourcesDir, 'package-flavor.json'), + `${JSON.stringify( + { + flavor, + appVersion: packageJson.version, + serverImage: await readServerImageMetadata(), + plugins, + commit: process.env.GITHUB_SHA ?? null, + ref: process.env.GITHUB_REF_NAME ?? null + }, + null, + 2 + )}\n` +) + +async function installPlugin({ id, label, repo, tag, artifact, variant = null }) { + const artifactPath = await resolveArtifact({ repo, tag, artifact }) + const target = join(preinstalledPluginDir, id) + + await rm(target, { recursive: true, force: true }) + await mkdir(target, { recursive: true }) + await extractZip(artifactPath, target) + await normalizePluginRoot(target) + + const pluginPackage = await validatePluginPackage(target, id) + plugins.push({ + id, + name: pluginPackage.pluginName, + version: pluginPackage.version ?? null, + label, + repo, + tag, + artifact, + variant + }) +} + +async function resolveArtifact({ repo, tag, artifact }) { + await mkdir(artifactDir, { recursive: true }) + + const artifactPath = join(artifactDir, artifact) + if (existsSync(artifactPath)) return artifactPath + + await run('gh', [ + 'release', + 'download', + tag, + '--repo', + repo, + '--pattern', + artifact, + '--dir', + artifactDir, + '--clobber' + ]) + + if (!existsSync(artifactPath)) { + throw new Error(`Expected release asset was not downloaded: ${repo} ${tag} ${artifact}`) + } + + return artifactPath +} + +async function normalizePluginRoot(target) { + if (existsSync(join(target, 'package.json'))) return + + const packageRoot = await findPackageRoot(target) + if (!packageRoot || packageRoot === target) return + + const normalizedTarget = `${target}.normalized` + await rm(normalizedTarget, { recursive: true, force: true }) + await rename(packageRoot, normalizedTarget) + await rm(target, { recursive: true, force: true }) + await rename(normalizedTarget, target) +} + +async function findPackageRoot(directory, depth = 0) { + if (existsSync(join(directory, 'package.json'))) return directory + if (depth >= 2) return null + + const entries = await readdir(directory, { withFileTypes: true }) + for (const entry of entries) { + if (!entry.isDirectory()) continue + + const found = await findPackageRoot(join(directory, entry.name), depth + 1) + if (found) return found + } + + return null +} + +async function validatePluginPackage(pluginRoot, expectedId) { + const packagePath = join(pluginRoot, 'package.json') + if (!existsSync(packagePath)) { + throw new Error(`Plugin artifact for ${expectedId} does not contain package.json`) + } + + const pluginPackage = JSON.parse(await readFile(packagePath, 'utf8')) + if (pluginPackage.name !== expectedId) { + throw new Error( + `Plugin artifact name mismatch: expected ${expectedId}, found ${pluginPackage.name}` + ) + } + + if (!pluginPackage.index || !existsSync(join(pluginRoot, pluginPackage.index))) { + throw new Error(`Plugin artifact for ${expectedId} does not contain ${pluginPackage.index}`) + } + + if ( + pluginPackage.dockerCompose && + !existsSync(join(pluginRoot, pluginPackage.dockerCompose)) + ) { + throw new Error( + `Plugin artifact for ${expectedId} does not contain ${pluginPackage.dockerCompose}` + ) + } + + return pluginPackage +} + +async function readServerImageMetadata() { + const serverImagePath = join(extraResourcesDir, 'server', 'server-image.json') + if (!existsSync(serverImagePath)) return null + return JSON.parse(await readFile(serverImagePath, 'utf8')) +} + +function resolvePathFromRoot(path) { + return isAbsolute(path) ? path : join(root, path) +} + +async function extractZip(artifactPath, target) { + if (process.platform === 'win32') { + await run('powershell', [ + '-NoProfile', + '-Command', + 'Expand-Archive -LiteralPath $args[0] -DestinationPath $args[1] -Force', + artifactPath, + target + ]) + return + } + + await run('unzip', ['-q', artifactPath, '-d', target]) +} + +async function run(command, args) { + await new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: 'inherit' }) + child.on('error', reject) + child.on('close', (code) => { + if (code === 0) { + resolve() + } else { + reject(new Error(`${command} ${args.join(' ')} exited with code ${code}`)) + } + }) + }) +} diff --git a/scripts/rename-release-artifacts.mjs b/scripts/rename-release-artifacts.mjs new file mode 100644 index 0000000..751f382 --- /dev/null +++ b/scripts/rename-release-artifacts.mjs @@ -0,0 +1,63 @@ +import { readFile, readdir, rename, writeFile } from 'node:fs/promises' +import { join } from 'node:path' + +const root = process.cwd() +const dist = join(root, 'dist') +const packageJson = JSON.parse(await readFile(join(root, 'package.json'), 'utf8')) +const flavor = process.env.OUROBOROS_PACKAGE_FLAVOR + +if (!flavor) { + console.log('OUROBOROS_PACKAGE_FLAVOR is not set; leaving release artifact names unchanged.') + process.exit(0) +} + +const entries = await readdir(dist, { withFileTypes: true }) +const renameMap = new Map() + +for (const entry of entries) { + if (!entry.isFile()) continue + + const nextName = artifactNameForFlavor(entry.name) + if (!nextName || nextName === entry.name) continue + + renameMap.set(entry.name, nextName) +} + +for (const [oldName, newName] of renameMap) { + await rename(join(dist, oldName), join(dist, newName)) + console.log(`Renamed ${oldName} -> ${newName}`) +} + +const updatedEntries = await readdir(dist, { withFileTypes: true }) +for (const entry of updatedEntries) { + if (!entry.isFile() || !entry.name.endsWith('.yml')) continue + + const path = join(dist, entry.name) + let text = await readFile(path, 'utf8') + + for (const [oldName, newName] of renameMap) { + text = text.split(oldName).join(newName) + } + + await writeFile(path, text) +} + +function artifactNameForFlavor(fileName) { + if (fileName.includes(`-${flavor}`) || fileName.includes(`_${flavor}`)) return null + + if (fileName.startsWith('latest') && fileName.endsWith('.yml')) { + return `${fileName.slice(0, -'.yml'.length)}-${flavor}.yml` + } + + const dashedPrefix = `${packageJson.name}-${packageJson.version}` + if (fileName.startsWith(dashedPrefix)) { + return `${dashedPrefix}-${flavor}${fileName.slice(dashedPrefix.length)}` + } + + const underscoredPrefix = `${packageJson.name}_${packageJson.version}` + if (fileName.startsWith(underscoredPrefix)) { + return `${underscoredPrefix}_${flavor}${fileName.slice(underscoredPrefix.length)}` + } + + return null +}