Build standalone Oatmeal desktop apps with local AI and provider choice

.github/workflows/desktop.yml

@@ -0,0 +1,88 @@
+name: Desktop checks and installers
+on:
+  push:
+    branches: [main]
+    tags: ['v*']
+  pull_request:
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  desktop:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: macos-14
+            target: mac
+            arch: arm64
+          - os: macos-15-intel
+            target: mac
+            arch: x64
+          - os: windows-latest
+            target: win
+            arch: x64
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+      - run: npm ci
+      - run: npm test
+      - run: npm run test:desktop
+      - name: Package development installers
+        if: ${{ !startsWith(github.ref, 'refs/tags/v') }}
+        run: npm run build -- --${{ matrix.target }} --${{ matrix.arch }} --publish never
+        env:
+          CSC_IDENTITY_AUTO_DISCOVERY: 'false'
+      - name: Check release signing credentials
+        if: startsWith(github.ref, 'refs/tags/v')
+        shell: bash
+        env:
+          MAC_CERT: ${{ secrets.MAC_CSC_LINK }}
+          WIN_CERT: ${{ secrets.WIN_CSC_LINK }}
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
+          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
+        run: |
+          if [[ '${{ matrix.target }}' == 'mac' ]]; then
+            test -n "$MAC_CERT" && test -n "$APPLE_ID" && test -n "$APPLE_APP_SPECIFIC_PASSWORD" && test -n "$APPLE_TEAM_ID" || { echo 'Missing macOS signing/notarization secrets'; exit 1; }
+          else
+            test -n "$WIN_CERT" || { echo 'Missing Windows signing certificate'; exit 1; }
+          fi
+      - name: Build signed release installers
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: npm run build -- --${{ matrix.target }} --${{ matrix.arch }} --publish never
+        env:
+          CSC_LINK: ${{ matrix.target == 'mac' && secrets.MAC_CSC_LINK || secrets.WIN_CSC_LINK }}
+          CSC_KEY_PASSWORD: ${{ matrix.target == 'mac' && secrets.MAC_CSC_KEY_PASSWORD || secrets.WIN_CSC_KEY_PASSWORD }}
+          APPLE_ID: ${{ secrets.APPLE_ID }}
+          APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
+          APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: oatmeal-${{ matrix.target }}-${{ matrix.arch }}
+          path: |
+            dist/*.dmg
+            dist/*.zip
+            dist/*.exe
+          if-no-files-found: error
+  release:
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs: desktop
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: installers
+          merge-multiple: true
+      - uses: softprops/action-gh-release@v2
+        with:
+          draft: true
+          files: installers/*
+          fail_on_unmatched_files: true

.gitignore

@@ -5,3 +5,10 @@ meetings/*
 !meetings/.gitkeep
 oatmeal.config.json
 .oatmeal-notified.json
+dist/
+test-results/
+playwright-report/
+.env*
+!.env.example
+*.p12
+*.pfx

AGENTS.md

@@ -0,0 +1,16 @@
+# Oatmeal desktop
+
+- Run `npm test` and `npm run test:desktop` after desktop behavior changes.
+- Keep API keys and model HTTP calls in the main process. The renderer receives only safe metadata through the preload bridge.
+- Preserve local models and personal API-key providers when changing built-in providers.
+- Do not read `.env` files or print authentication credentials.
+
+## publik API
+- App slug: `oatmeal-vedsoni-dev` (the `oatmeal` listing belongs to another repository)
+- Base URL: https://publikhq.com/api/v1 (OpenAI Chat Completions, Responses, Anthropic Messages)
+- Models: publik-fast, publik-balanced, publik-smart only; never a vendor model name
+- Key: per-install `pk_` from POST /installs, stored at the publik credential path; env PUBLIK_API_KEY / PUBLIK_API_BASE_URL override
+- App token: publik-app-token.txt (public by design, committed) or PUBLIK_APP_TOKEN
+- 402: show error.message plus error.top_up_url only
+- Never print or log a key
+- Docs: https://publikhq.com/llms.txt, https://publikhq.com/developers, https://publikhq.com/skills/api.md

SKILL.md

@@ -24,7 +24,7 @@ doesn't matter.
 2. Run `node scripts/install-autostart.mjs`. This registers the recorder as an
    OS-level background service (Windows Scheduled Task / macOS LaunchAgent / Linux
    systemd --user service), starts it immediately, and makes it auto-start at every
-   login from now on. You do not need to run `npm start` manually, and you do not
+   login from now on. You do not need to run `npm run start:browser` manually, and you do not
    need to stay open for the recorder to work tomorrow.
 3. Open http://localhost:4123 in the user's default browser
    (Windows: `start http://localhost:4123`, macOS: `open`, Linux: `xdg-open`).

build/entitlements.mac.plist

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0"><dict>
+  <key>com.apple.security.cs.allow-jit</key><true/>
+  <key>com.apple.security.cs.allow-unsigned-executable-memory</key><true/>
+  <key>com.apple.security.device.audio-input</key><true/>
+</dict></plist>

capture/public/app.js

@@ -80,11 +80,11 @@ async function loadWhisper() {
   const want = modelSelect.value
   if (asr && asrModel === want) return asr
   asr = null
-  const { pipeline, env } = await import('/vendor/transformers/transformers.min.js')
+  const { pipeline, env } = await import('/vendor/transformers/transformers.web.min.js')
   env.allowLocalModels = false
   if (env.backends?.onnx?.wasm) {
     env.backends.onnx.wasm.numThreads = 1
-    env.backends.onnx.wasm.wasmPaths = new URL('/vendor/transformers/', location.href).href
+    env.backends.onnx.wasm.wasmPaths = new URL('/vendor/onnx/', location.href).href
   }
   const device = 'gpu' in navigator ? 'webgpu' : 'wasm'
   const make = (dev) =>

capture/server.mjs

@@ -32,6 +32,7 @@ const MIME = {
 // Static roots: the UI, plus transformers.js + onnx runtime from node_modules so
 // Whisper runs fully local (no CDN).
 const STATIC = [
+  { prefix: '/vendor/onnx/', dir: join(ROOT, 'node_modules/onnxruntime-web/dist') },
   { prefix: '/vendor/transformers/', dir: join(ROOT, 'node_modules/@huggingface/transformers/dist') },
   { prefix: '/', dir: join(__dirname, 'public') }
 ]
@@ -157,7 +158,7 @@ server.on('error', (e) => {
   throw e
 })
 
-server.listen(PORT, () => {
+server.listen(PORT, '127.0.0.1', () => {
   console.log(`[oatmeal] capture server on http://localhost:${PORT}`)
   console.log(`[oatmeal] transcripts land in ${MEETINGS_DIR}`)
 })

desktop/core/inference.mjs

@@ -0,0 +1,52 @@
+import { Worker } from 'node:worker_threads'
+
+export class Inference {
+  constructor(cacheDir, onProgress = () => {}) {
+    this.cacheDir = cacheDir
+    this.onProgress = onProgress
+    this.pending = new Map()
+    this.nextId = 0
+  }
+  start() {
+    if (this.worker) return
+    this.worker = new Worker(
+      new URL('../workers/inference.mjs', import.meta.url),
+      { workerData: { cacheDir: this.cacheDir }, execArgv: [] },
+    )
+    const worker = this.worker
+    this.worker.on('message', (message) => {
+      if (message.event === 'progress') {
+        this.onProgress(message)
+        return
+      }
+      const task = this.pending.get(message.id)
+      if (!task) return
+      this.pending.delete(message.id)
+      message.error
+        ? task.reject(new Error(message.error))
+        : task.resolve(message.result)
+    })
+    const fail = () => {
+      if (this.worker !== worker) return
+      for (const task of this.pending.values())
+        task.reject(
+          new Error('The local model worker stopped. Retry the operation.'),
+        )
+      this.pending.clear()
+      this.worker = null
+    }
+    this.worker.on('error', fail)
+    this.worker.on('exit', fail)
+  }
+  run(command, data = {}) {
+    this.start()
+    const id = ++this.nextId
+    return new Promise((resolve, reject) => {
+      this.pending.set(id, { resolve, reject })
+      this.worker.postMessage({ id, command, ...data })
+    })
+  }
+  async close() {
+    if (this.worker) await this.worker.terminate()
+  }
+}

desktop/core/local-notes.mjs

@@ -0,0 +1,146 @@
+// Small local models can invent elaborate decisions from very short input.
+// Let them select evidence, then render the original words, never generated facts.
+export function evidence(meeting) {
+  const rows = []
+  const add = (source, value) => {
+    for (const text of value
+      .split(/(?<=[.!?])\s+|\n+/)
+      .map((s) => s.trim())
+      .filter(Boolean)) {
+      // Keep every part of unusually long utterances without unbounded prompts.
+      for (let at = 0; at < text.length; at += 1000)
+        rows.push({
+          source,
+          text: text.slice(at, at + 1000),
+          id: rows.length + 1,
+        })
+    }
+  }
+  add('Your notes', meeting.notes)
+  for (const segment of meeting.segments)
+    add(
+      `${segment.speaker === 'you' ? 'You' : 'Room'}, ${Math.floor(segment.offset / 60)}:${String(Math.floor(segment.offset % 60)).padStart(2, '0')}`,
+      segment.text,
+    )
+  return rows
+}
+
+function groups(rows, limit = 6000) {
+  const result = []
+  let group = [],
+    length = 0
+  for (const row of rows) {
+    if (group.length && length + row.text.length > limit) {
+      result.push(group)
+      group = []
+      length = 0
+    }
+    group.push(row)
+    length += row.text.length + 40
+  }
+  if (group.length) result.push(group)
+  return result
+}
+
+function selectedIds(output, allowed) {
+  // Only accept standalone list lines. Prose, invented names, and rationale never
+  // enter the document. Out-of-range identifiers are discarded.
+  const ids =
+    output
+      .split('\n')
+      .filter((line) => /^\s*[\d,\s\[\]-]+\s*$/.test(line))
+      .join(' ')
+      .match(/\d+/g) || []
+  return [...new Set(ids.map(Number))].filter((id) => allowed.has(id))
+}
+const render = (rows) =>
+  rows.map((row) => `- ${row.text}\n  (${row.source})`).join('\n')
+function keywordMatches(rows, question) {
+  const ignored = new Set([
+    'the',
+    'and',
+    'who',
+    'what',
+    'when',
+    'where',
+    'why',
+    'how',
+    'did',
+    'does',
+    'was',
+    'were',
+    'with',
+    'this',
+    'that',
+    'for',
+    'will',
+    'can',
+    'about',
+    'from',
+  ])
+  const words = (value) =>
+    value
+      .toLowerCase()
+      .match(/[\p{L}\p{N}]+/gu)
+      ?.map((word) => word.replace(/s$/, ''))
+      .filter((word) => word.length > 2 && !ignored.has(word)) || []
+  const terms = new Set(words(question))
+  return rows
+    .map((row) => ({
+      row,
+      score: words(row.text).filter((word) => terms.has(word)).length,
+    }))
+    .filter((hit) => hit.score > 0)
+    .sort((a, b) => b.score - a.score)
+    .slice(0, 5)
+    .map((hit) => hit.row)
+}
+
+export async function localNotes(meeting, complete) {
+  const rows = evidence(meeting)
+  if (!rows.length) throw new Error('Add notes or record a transcript first.')
+  const selected = []
+  for (const group of groups(rows)) {
+    const output = await complete(
+      'Select the most important statements from this meeting. Reply only with up to 10 statement numbers, separated by commas. Never write new statements.',
+      group.map((row) => `${row.id}. ${row.text}`).join('\n'),
+    )
+    const ids = selectedIds(output, new Set(group.map((row) => row.id)))
+    selected.push(
+      ...(ids.length ? group.filter((row) => ids.includes(row.id)) : group),
+    )
+  }
+  const actions = rows.filter((row) =>
+    /\b(will|i'll|we'll|assigned|action item|to.do|by (monday|tuesday|wednesday|thursday|friday|tomorrow))\b/i.test(
+      row.text,
+    ),
+  )
+  const decisions = rows.filter((row) =>
+    /\b(decided|agreed|approved|decision|we chose|we selected)\b/i.test(
+      row.text,
+    ),
+  )
+  return `## Key excerpts\n\n${render(selected)}\n\n## Decision excerpts\n\n${decisions.length ? render(decisions) : 'No explicit decision wording found. Review the transcript for context.'}\n\n## Possible follow-ups\n\n${actions.length ? render(actions) : 'No explicit follow-up wording found. Review the transcript for context.'}\n\nLocal notes use original wording to preserve facts. Follow-ups are suggestions to review, not confirmed assignments.`
+}
+
+export async function localAnswer(meeting, question, complete) {
+  const rows = evidence(meeting)
+  if (!rows.length)
+    throw new Error('This meeting has no notes or transcript yet.')
+  const selected = []
+  for (const group of groups(rows)) {
+    const output = await complete(
+      'Find meeting statements relevant to the question. Reply only with up to 5 statement numbers separated by commas. Reply 0 if none are relevant. Do not answer from memory.',
+      `Question: ${question}\n\n${group.map((row) => `${row.id}. ${row.text}`).join('\n')}`,
+    )
+    const ids = selectedIds(output, new Set(group.map((row) => row.id)))
+    selected.push(
+      ...(ids.length
+        ? group.filter((row) => ids.includes(row.id))
+        : keywordMatches(group, question)),
+    )
+  }
+  return selected.length
+    ? `## Relevant meeting excerpts\n\n${render(selected)}\n\nThese are original meeting statements; check their context in the transcript.`
+    : 'The local model did not find supporting excerpts. Try different wording or review the transcript.'
+}