Automatically deploy each main detector build to production and recovery

.github/workflows/hosting-runtime.yml

@@ -1,32 +1,37 @@
-name: Build pinned hosting runtime
+name: Build and deploy hosting runtime
 
 on:
   push:
     branches: [main]
+  pull_request:
     paths:
-      - k8s/local-hosting/**
-      - observability.py
-      - main.py
-      - utils/model_download.py
+      - scripts/hosting_*.py
+      - scripts/test_hosting_release.py
+      - scripts/wait_hosting_release.py
       - .github/workflows/hosting-runtime.yml
-  workflow_dispatch: {}
 
 permissions:
   contents: read
   packages: write
+  statuses: read
 
 jobs:
   runtime:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Verify release coordination
+        run: python3 -m unittest discover -s scripts -p 'test_hosting_release.py' -v
       - uses: docker/setup-buildx-action@v3
       - uses: docker/login-action@v3
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - uses: docker/build-push-action@v6
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        id: build
         with:
           context: .
           file: k8s/local-hosting/Dockerfile
@@ -35,3 +40,29 @@ jobs:
           platforms: linux/amd64
           push: true
           tags: ghcr.io/opensensor/light-object-detect:hosting-${{ github.sha }}
+      - name: Record this build's platform image
+        if: steps.build.outcome == 'success'
+        env:
+          BUILD_DIGEST: ${{ steps.build.outputs.digest }}
+        run: python3 scripts/hosting_release_manifest.py
+      - uses: actions/upload-artifact@v4
+        if: steps.build.outcome == 'success'
+        with:
+          name: hosting-release-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: hosting-release.json
+          retention-days: 30
+          if-no-files-found: error
+  deploy:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: runtime
+    concurrency:
+      group: detector-production-verification
+      cancel-in-progress: true
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v4
+      - name: Verify this build is deployed to production and recovery
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python3 scripts/wait_hosting_release.py

scripts/deploy/README.md

@@ -0,0 +1,29 @@
+# Automatic paired detector deployment
+
+Every main push builds the hosting image and publishes a receipt containing its
+source revision, workflow run and immutable amd64 image digest. The operator host
+polls for successful builds every minute using its existing GitHub and cluster
+access. CI waits for a status specific to that build and fails if deployment is
+not verified. Pull requests test coordination without deploying.
+
+The agent warms cloud, moves traffic there, updates cloud and local, updates the
+recovery controller's expected image, returns traffic locally, and verifies the
+cloud persistent model cache before scaling cloud back to zero. Checkpoints make
+interrupted handoffs resumable. A new build waits for an active handoff; an older
+build cannot replace a newer completed release. Operator holds are preserved.
+
+Install once on the existing operator host (no new credentials):
+
+```sh
+install -Dm755 scripts/deploy/detector-release-launch.sh ~/.local/lib/opensensor-hosting/detector-release-launch.sh
+install -Dm644 scripts/deploy/opensensor-detector-release.service ~/.config/systemd/user/opensensor-detector-release.service
+install -Dm644 scripts/deploy/opensensor-detector-release.timer ~/.config/systemd/user/opensensor-detector-release.timer
+systemctl --user daemon-reload
+systemctl --user enable --now opensensor-detector-release.timer
+```
+
+The launcher follows main. Receipts and resumable state are private files under
+`~/.config/opensensor-hosting/detector-release/`. The progress snapshot includes
+existing deployment configuration; keep that directory private. Review failures
+with `journalctl --user -u opensensor-detector-release.service`. The timer retries
+a failed handoff; do not delete pending state during a deployment.

scripts/deploy/detector-release-launch.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+release_checkout="$HOME/.cache/opensensor-hosting/detector-release"
+if [[ ! -d "$release_checkout/.git" ]]; then
+    git clone --branch main --single-branch https://github.com/opensensor/light-object-detect.git "$release_checkout"
+fi
+git -C "$release_checkout" fetch origin main
+git -C "$release_checkout" merge --ff-only origin/main
+agent="$release_checkout/scripts/hosting_release.py"
+[[ -f "$agent" ]] || exit 0
+exec /usr/bin/python3 "$agent" --apply

scripts/deploy/opensensor-detector-release.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Deploy latest main detector build to production and recovery
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+Environment=PATH=/snap/bin:/usr/local/bin:/usr/bin:/bin
+ExecStart=/usr/bin/bash %h/.local/lib/opensensor-hosting/detector-release-launch.sh
+TimeoutStartSec=35min
+UMask=0077
+Nice=5

scripts/deploy/opensensor-detector-release.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Watch for new main detector images
+
+[Timer]
+OnBootSec=30s
+OnUnitInactiveSec=1min
+AccuracySec=5s
+Unit=opensensor-detector-release.service
+
+[Install]
+WantedBy=timers.target

scripts/hosting_release.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Consume the latest successful main image build and resume paired deployments."""
+import argparse
+import fcntl
+import io
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import time
+import zipfile
+
+from hosting_runtime import deploy, save_progress
+
+REPO = 'opensensor/light-object-detect'
+WORKFLOW = '.github/workflows/hosting-runtime.yml'
+CONTEXT = 'hosting/detector-release'
+STATE = Path.home() / '.config/opensensor-hosting/detector-release'
+IMAGE = re.compile(r'ghcr\.io/opensensor/light-object-detect@sha256:[a-f0-9]{64}')
+
+
+def gh(path, **fields):
+    args = ['gh', 'api', '-H', 'Cache-Control: no-cache', path]
+    for key, value in fields.items():args += ['-f', key + '=' + str(value)]
+    return json.loads(subprocess.check_output(args, timeout=90))
+
+
+def validate(target, workflow):
+    if (workflow['event'] != 'push' or workflow['head_branch'] != 'main'
+            or workflow['path'] != WORKFLOW or workflow['repository']['full_name'] != REPO
+            or set(target) != {'format', 'repository', 'commit', 'run_id', 'run_attempt', 'image'}
+            or target['format'] != 'detector-release-v1' or target['repository'] != REPO
+            or target['commit'] != workflow['head_sha'] or target['run_id'] != workflow['id']
+            or target['run_attempt'] != workflow['run_attempt']
+            or not re.fullmatch('[a-f0-9]{40}', target['commit']) or not IMAGE.fullmatch(target['image'])):
+        raise ValueError('Release must identify this main build and its immutable image')
+    return target
+
+
+def latest():
+    runs = gh(f'repos/{REPO}/actions/workflows/hosting-runtime.yml/runs?branch=main&event=push&per_page=20')['workflow_runs']
+    for workflow in sorted(runs, key=lambda r:(r['run_number'], r['run_attempt']), reverse=True):
+        jobs = gh(f'repos/{REPO}/actions/runs/{workflow["id"]}/attempts/{workflow["run_attempt"]}/jobs?per_page=100')['jobs']
+        if not any(j['name']=='runtime' and j['conclusion']=='success' for j in jobs):continue
+        artifacts = gh(f'repos/{REPO}/actions/runs/{workflow["id"]}/artifacts?per_page=100')['artifacts']
+        name = f'hosting-release-{workflow["head_sha"]}-{workflow["id"]}-{workflow["run_attempt"]}'
+        matches = [a for a in artifacts if a['name']==name and not a['expired']]
+        if len(matches)!=1:raise RuntimeError('Successful build is missing its release artifact')
+        raw = subprocess.check_output(['gh','api',f'repos/{REPO}/actions/artifacts/{matches[0]["id"]}/zip'],timeout=90)
+        with zipfile.ZipFile(io.BytesIO(raw)) as archive:
+            if archive.namelist()!=['hosting-release.json'] or archive.infolist()[0].file_size>8192:
+                raise ValueError('Unexpected release artifact')
+            return validate(json.loads(archive.read('hosting-release.json')),workflow)
+    return None
+
+
+def report(target, state, description):
+    gh(f'repos/{REPO}/statuses/{target["commit"]}',state=state,
+       context=f'{CONTEXT}/{target["run_id"]}/{target["run_attempt"]}',description=description[:140],
+       target_url=f'https://github.com/{REPO}/actions/runs/{target["run_id"]}')
+
+
+def choose(pending, candidate, previous):
+    # Finish a handoff already in progress before starting another release.
+    if pending:return pending
+    if not candidate:return None
+    if previous:
+        before=(previous['run_id'],previous['run_attempt'])
+        after=(candidate['run_id'],candidate['run_attempt'])
+        if after<before:raise RuntimeError('Refuse an older release')
+        if after==before and previous.get('state')=='success':return None
+    return candidate
+
+
+def main():
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--apply',action='store_true')
+    args=parser.parse_args()
+    os.umask(0o077);STATE.mkdir(parents=True,exist_ok=True)
+    with (STATE/'agent.lock').open('a') as lock:
+        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+        read=lambda name:json.loads((STATE/name).read_text()) if (STATE/name).exists() else {}
+        pending, previous=read('pending.json'),read('latest.json')
+        target=choose(pending,None if pending else latest(),previous)
+        if not target:return
+        if not args.apply:print(json.dumps(target));return
+        save_progress(STATE/'pending.json',target)
+        directory=STATE/f'{target["run_id"]}-{target["run_attempt"]}'
+        directory.mkdir(exist_ok=True)
+        report(target,'pending','Automatically deploying this build to both detector origins')
+        try:
+            result=deploy(target,directory)
+            receipt={**target,**result,'state':'success','updated_at':time.time()}
+            report(target,'success','Exact image serving locally; cloud recovery image and persistent cache verified')
+            save_progress(STATE/'latest.json',receipt)
+            (STATE/'pending.json').unlink()
+        except Exception as error:
+            save_progress(STATE/'latest.json',{**target,'state':'failure','error_type':type(error).__name__,'updated_at':time.time()})
+            report(target,'failure','Automatic deployment will retry: '+type(error).__name__)
+            raise
+
+
+if __name__=='__main__':main()

scripts/hosting_release_manifest.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""Bind a release to this build's amd64 manifest, excluding provenance attestations."""
+import json
+import os
+import subprocess
+
+
+def platform_digest(document, digest):
+    if 'manifests' not in document:return digest
+    matches=[m['digest'] for m in document['manifests'] if m.get('platform')=={'architecture':'amd64','os':'linux'}]
+    if len(matches)!=1:raise ValueError('Expected exactly one linux/amd64 image')
+    return matches[0]
+
+
+def main():
+    digest=os.environ['BUILD_DIGEST']
+    repository='ghcr.io/opensensor/light-object-detect'
+    document=json.loads(subprocess.check_output(['docker','buildx','imagetools','inspect',repository+'@'+digest,'--raw']))
+    target={'format':'detector-release-v1','repository':os.environ['GITHUB_REPOSITORY'],
+        'commit':os.environ['GITHUB_SHA'],'run_id':int(os.environ['GITHUB_RUN_ID']),
+        'run_attempt':int(os.environ['GITHUB_RUN_ATTEMPT']),
+        'image':repository+'@'+platform_digest(document,digest)}
+    with open('hosting-release.json','w') as out:json.dump(target,out)
+
+
+if __name__=='__main__':main()