Build production package flavors

.github/workflows/build-and-release.yml

@@ -4,9 +4,32 @@ on:
   push:
     tags:
       - v*.*.*
+  workflow_dispatch:
+    inputs:
+      server_image_tag:
+        description: Server image tag to package. Defaults to the release tag.
+        required: false
+      server_image_digest:
+        description: Optional server image digest. Overrides the tag when set.
+        required: false
+      neuroglancer_plugin_tag:
+        description: Neuroglancer plugin release tag. Defaults to the release tag.
+        required: false
+      neuroglancer_plugin_artifact:
+        description: Neuroglancer plugin artifact name. Defaults from the plugin tag.
+        required: false
+      autoseg_plugin_tag:
+        description: Automatic segmentation plugin release tag. Defaults to the release tag.
+        required: false
+      autoseg_cpu_plugin_artifact:
+        description: CPU automatic segmentation plugin artifact name. Defaults from the plugin tag.
+        required: false
+      autoseg_cuda_plugin_artifact:
+        description: CUDA automatic segmentation plugin artifact name. Defaults from the plugin tag.
+        required: false
 
 env: 
-  GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+  GH_TOKEN: "${{ secrets.OUROBOROS_RELEASE_ASSET_TOKEN || secrets.GITHUB_TOKEN }}"
   GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
 
 jobs:
@@ -19,6 +42,10 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
+        package:
+          - flavor: core
+          - flavor: plugins-cpu
+          - flavor: plugins-cuda
 
     steps:
       - name: Check out Git repository
@@ -35,7 +62,18 @@ jobs:
       - name: Prepare registry-backed server resources
         run: npm run prepare:production-server
         env:
-          OUROBOROS_SERVER_IMAGE_TAG: ${{ github.ref_name }}
+          OUROBOROS_SERVER_IMAGE_TAG: ${{ github.event.inputs.server_image_tag || github.ref_name }}
+          OUROBOROS_SERVER_IMAGE_DIGEST: ${{ github.event.inputs.server_image_digest }}
+
+      - name: Prepare production package flavor
+        run: npm run prepare:package-flavor
+        env:
+          OUROBOROS_PACKAGE_FLAVOR: ${{ matrix.package.flavor }}
+          OUROBOROS_NEUROGLANCER_PLUGIN_TAG: ${{ github.event.inputs.neuroglancer_plugin_tag || github.ref_name }}
+          OUROBOROS_NEUROGLANCER_PLUGIN_ARTIFACT: ${{ github.event.inputs.neuroglancer_plugin_artifact }}
+          OUROBOROS_AUTOSEG_PLUGIN_TAG: ${{ github.event.inputs.autoseg_plugin_tag || github.ref_name }}
+          OUROBOROS_AUTOSEG_CPU_PLUGIN_ARTIFACT: ${{ github.event.inputs.autoseg_cpu_plugin_artifact }}
+          OUROBOROS_AUTOSEG_CUDA_PLUGIN_ARTIFACT: ${{ github.event.inputs.autoseg_cuda_plugin_artifact }}
 
       - name: build-linux
         if: matrix.os == 'ubuntu-latest'
@@ -49,6 +87,11 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: npm run build:win
 
+      - name: Add package flavor to artifact names
+        run: npm run rename:release-artifacts
+        env:
+          OUROBOROS_PACKAGE_FLAVOR: ${{ matrix.package.flavor }}
+
       - name: release
         uses: softprops/action-gh-release@v1
         with:

.gitignore

@@ -5,6 +5,7 @@ out
 *.log*
 documentation/site
 extra-resources
+.package-plugin-artifacts
 data
 ignoreme.py
 temp

documentation/docs/development/production-package-flavors.md

@@ -0,0 +1,52 @@
+# Production package flavors
+
+The release workflow builds three production package flavors for each supported
+desktop runner:
+
+- `core`: Ouroboros only, with the production server compose file pointing at
+  the registry-published server image.
+- `plugins-cpu`: the core package plus bundled Neuroglancer and Automatic
+  Segmentation plugin artifacts. The automatic segmentation plugin uses its CPU
+  backend image.
+- `plugins-cuda`: the core package plus bundled Neuroglancer and Automatic
+  Segmentation plugin artifacts. The automatic segmentation plugin uses its
+  CUDA backend image and Docker Compose GPU reservation.
+
+Plugin flavors stage release artifacts into
+`extra-resources/preinstalled-plugins` before Electron Builder runs. The app
+installs those bundled plugin folders into the user plugin directory on startup,
+so production packages can ship with plugins already available.
+
+## Release inputs
+
+Tag releases default every image and plugin artifact to the Ouroboros release
+tag. Manual workflow runs can override these inputs:
+
+- `server_image_tag` or `server_image_digest`
+- `neuroglancer_plugin_tag`
+- `neuroglancer_plugin_artifact`
+- `autoseg_plugin_tag`
+- `autoseg_cpu_plugin_artifact`
+- `autoseg_cuda_plugin_artifact`
+
+The default plugin artifact names are:
+
+- `neuroglancer-plugin-<tag>.zip`
+- `auto-segmentation-<tag>-cpu.zip`
+- `auto-segmentation-<tag>-cuda.zip`
+
+If plugin release repositories are private, set
+`OUROBOROS_RELEASE_ASSET_TOKEN` to a token that can read those release assets.
+The workflow falls back to `GITHUB_TOKEN` when that secret is not set.
+
+## Local checks
+
+Prepare the core package resources with:
+
+```sh
+npm run prepare:production-server
+OUROBOROS_PACKAGE_FLAVOR=core npm run prepare:package-flavor
+```
+
+Plugin flavors can be checked against local release zips by placing them in
+`.package-plugin-artifacts` or setting `OUROBOROS_PLUGIN_ARTIFACT_DIR`.

documentation/mkdocs.yml

@@ -17,6 +17,7 @@ nav:
     - Plugins: https://github.com/ChengLabResearch/ouroboros/blob/main/plugins/plugin-template/README.md
     - Docker Builds: 'development/docker-builds.md'
     - Registry Server Images: 'development/registry-server-images.md'
+    - Production Package Flavors: 'development/production-package-flavors.md'
   - Releases: 'https://github.com/ChengLabResearch/ouroboros/releases'
   - Repository: https://github.com/ChengLabResearch/ouroboros
 

package.json

@@ -16,6 +16,8 @@
 		"dev": "electron-vite dev",
 		"build": "npm run typecheck && electron-vite build",
 		"prepare:production-server": "node scripts/prepare-production-server.mjs",
+		"prepare:package-flavor": "node scripts/prepare-package-flavor.mjs",
+		"rename:release-artifacts": "node scripts/rename-release-artifacts.mjs",
 		"postinstall": "electron-builder install-app-deps",
 		"build:unpack": "npm run build && electron-builder --dir",
 		"build:win": "electron-vite build && electron-builder --win --config --publish never",

scripts/prepare-package-flavor.mjs

@@ -0,0 +1,217 @@
+import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
+import { isAbsolute, join } from 'node:path'
+import { spawn } from 'node:child_process'
+
+const supportedFlavors = new Set(['core', 'plugins-cpu', 'plugins-cuda'])
+
+const root = process.cwd()
+const packageJson = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'))
+const flavor = process.env.OUROBOROS_PACKAGE_FLAVOR ?? 'core'
+
+if (!supportedFlavors.has(flavor)) {
+	throw new Error(
+		`Unsupported OUROBOROS_PACKAGE_FLAVOR "${flavor}". Expected one of: ${[
+			...supportedFlavors
+		].join(', ')}`
+	)
+}
+
+const extraResourcesDir = join(root, 'extra-resources')
+const preinstalledPluginDir = join(extraResourcesDir, 'preinstalled-plugins')
+const artifactDir = resolvePathFromRoot(
+	process.env.OUROBOROS_PLUGIN_ARTIFACT_DIR ?? '.package-plugin-artifacts'
+)
+const releaseTag = process.env.GITHUB_REF_NAME ?? `v${packageJson.version}`
+const plugins = []
+
+await mkdir(extraResourcesDir, { recursive: true })
+await rm(preinstalledPluginDir, { recursive: true, force: true })
+
+if (flavor !== 'core') {
+	await mkdir(preinstalledPluginDir, { recursive: true })
+
+	const neuroglancerTag = process.env.OUROBOROS_NEUROGLANCER_PLUGIN_TAG || releaseTag
+	await installPlugin({
+		id: 'neuroglancer-plugin',
+		label: 'Neuroglancer',
+		repo: 'ChengLabResearch/neuroglancer-plugin',
+		tag: neuroglancerTag,
+		artifact:
+			process.env.OUROBOROS_NEUROGLANCER_PLUGIN_ARTIFACT ||
+			`neuroglancer-plugin-${neuroglancerTag}.zip`
+	})
+
+	const autosegTag = process.env.OUROBOROS_AUTOSEG_PLUGIN_TAG || releaseTag
+	const autosegVariant = flavor === 'plugins-cuda' ? 'cuda' : 'cpu'
+	await installPlugin({
+		id: 'auto-segmentation',
+		label: `Automatic Segmentation (${autosegVariant.toUpperCase()})`,
+		repo: 'ChengLabResearch/ouroboros_autoseg_plugin',
+		tag: autosegTag,
+		artifact:
+			process.env[`OUROBOROS_AUTOSEG_${autosegVariant.toUpperCase()}_PLUGIN_ARTIFACT`] ||
+			`auto-segmentation-${autosegTag}-${autosegVariant}.zip`,
+		variant: autosegVariant
+	})
+}
+
+await writeFile(
+	join(extraResourcesDir, 'package-flavor.json'),
+	`${JSON.stringify(
+		{
+			flavor,
+			appVersion: packageJson.version,
+			serverImage: await readServerImageMetadata(),
+			plugins,
+			commit: process.env.GITHUB_SHA ?? null,
+			ref: process.env.GITHUB_REF_NAME ?? null
+		},
+		null,
+		2
+	)}\n`
+)
+
+async function installPlugin({ id, label, repo, tag, artifact, variant = null }) {
+	const artifactPath = await resolveArtifact({ repo, tag, artifact })
+	const target = join(preinstalledPluginDir, id)
+
+	await rm(target, { recursive: true, force: true })
+	await mkdir(target, { recursive: true })
+	await extractZip(artifactPath, target)
+	await normalizePluginRoot(target)
+
+	const pluginPackage = await validatePluginPackage(target, id)
+	plugins.push({
+		id,
+		name: pluginPackage.pluginName,
+		version: pluginPackage.version ?? null,
+		label,
+		repo,
+		tag,
+		artifact,
+		variant
+	})
+}
+
+async function resolveArtifact({ repo, tag, artifact }) {
+	await mkdir(artifactDir, { recursive: true })
+
+	const artifactPath = join(artifactDir, artifact)
+	if (existsSync(artifactPath)) return artifactPath
+
+	await run('gh', [
+		'release',
+		'download',
+		tag,
+		'--repo',
+		repo,
+		'--pattern',
+		artifact,
+		'--dir',
+		artifactDir,
+		'--clobber'
+	])
+
+	if (!existsSync(artifactPath)) {
+		throw new Error(`Expected release asset was not downloaded: ${repo} ${tag} ${artifact}`)
+	}
+
+	return artifactPath
+}
+
+async function normalizePluginRoot(target) {
+	if (existsSync(join(target, 'package.json'))) return
+
+	const packageRoot = await findPackageRoot(target)
+	if (!packageRoot || packageRoot === target) return
+
+	const normalizedTarget = `${target}.normalized`
+	await rm(normalizedTarget, { recursive: true, force: true })
+	await rename(packageRoot, normalizedTarget)
+	await rm(target, { recursive: true, force: true })
+	await rename(normalizedTarget, target)
+}
+
+async function findPackageRoot(directory, depth = 0) {
+	if (existsSync(join(directory, 'package.json'))) return directory
+	if (depth >= 2) return null
+
+	const entries = await readdir(directory, { withFileTypes: true })
+	for (const entry of entries) {
+		if (!entry.isDirectory()) continue
+
+		const found = await findPackageRoot(join(directory, entry.name), depth + 1)
+		if (found) return found
+	}
+
+	return null
+}
+
+async function validatePluginPackage(pluginRoot, expectedId) {
+	const packagePath = join(pluginRoot, 'package.json')
+	if (!existsSync(packagePath)) {
+		throw new Error(`Plugin artifact for ${expectedId} does not contain package.json`)
+	}
+
+	const pluginPackage = JSON.parse(await readFile(packagePath, 'utf8'))
+	if (pluginPackage.name !== expectedId) {
+		throw new Error(
+			`Plugin artifact name mismatch: expected ${expectedId}, found ${pluginPackage.name}`
+		)
+	}
+
+	if (!pluginPackage.index || !existsSync(join(pluginRoot, pluginPackage.index))) {
+		throw new Error(`Plugin artifact for ${expectedId} does not contain ${pluginPackage.index}`)
+	}
+
+	if (
+		pluginPackage.dockerCompose &&
+		!existsSync(join(pluginRoot, pluginPackage.dockerCompose))
+	) {
+		throw new Error(
+			`Plugin artifact for ${expectedId} does not contain ${pluginPackage.dockerCompose}`
+		)
+	}
+
+	return pluginPackage
+}
+
+async function readServerImageMetadata() {
+	const serverImagePath = join(extraResourcesDir, 'server', 'server-image.json')
+	if (!existsSync(serverImagePath)) return null
+	return JSON.parse(await readFile(serverImagePath, 'utf8'))
+}
+
+function resolvePathFromRoot(path) {
+	return isAbsolute(path) ? path : join(root, path)
+}
+
+async function extractZip(artifactPath, target) {
+	if (process.platform === 'win32') {
+		await run('powershell', [
+			'-NoProfile',
+			'-Command',
+			'Expand-Archive -LiteralPath $args[0] -DestinationPath $args[1] -Force',
+			artifactPath,
+			target
+		])
+		return
+	}
+
+	await run('unzip', ['-q', artifactPath, '-d', target])
+}
+
+async function run(command, args) {
+	await new Promise((resolve, reject) => {
+		const child = spawn(command, args, { stdio: 'inherit' })
+		child.on('error', reject)
+		child.on('close', (code) => {
+			if (code === 0) {
+				resolve()
+			} else {
+				reject(new Error(`${command} ${args.join(' ')} exited with code ${code}`))
+			}
+		})
+	})
+}