From 7de2224c72664e580cb4aa29256084d5e4605634 Mon Sep 17 00:00:00 2001
From: Matt Davis <matteius@gmail.com>
Date: Fri, 30 Oct 2026 10:40:35 -0400
Subject: [PATCH] Deploy every main detector build to both production origins

---
 .github/workflows/hosting-runtime.yml         |  43 ++++-
 scripts/deploy/README.md                      |  29 +++
 scripts/deploy/detector-release-launch.sh     |  11 ++
 .../opensensor-detector-release.service       |  12 ++
 .../deploy/opensensor-detector-release.timer  |  11 ++
 scripts/hosting_release.py                    | 104 +++++++++++
 scripts/hosting_release_manifest.py           |  26 +++
 scripts/hosting_runtime.py                    | 176 ++++++++++++++++++
 scripts/test_hosting_release.py               |  86 +++++++++
 scripts/wait_hosting_release.py               |  27 +++
 10 files changed, 519 insertions(+), 6 deletions(-)
 create mode 100644 scripts/deploy/README.md
 create mode 100644 scripts/deploy/detector-release-launch.sh
 create mode 100644 scripts/deploy/opensensor-detector-release.service
 create mode 100644 scripts/deploy/opensensor-detector-release.timer
 create mode 100644 scripts/hosting_release.py
 create mode 100644 scripts/hosting_release_manifest.py
 create mode 100644 scripts/hosting_runtime.py
 create mode 100644 scripts/test_hosting_release.py
 create mode 100644 scripts/wait_hosting_release.py

diff --git a/.github/workflows/hosting-runtime.yml b/.github/workflows/hosting-runtime.yml
index 104fa67..bb64155 100644
--- a/.github/workflows/hosting-runtime.yml
+++ b/.github/workflows/hosting-runtime.yml
@@ -1,32 +1,37 @@
-name: Build pinned hosting runtime
+name: Build and deploy hosting runtime
 
 on:
   push:
     branches: [main]
+  pull_request:
     paths:
-      - k8s/local-hosting/**
-      - observability.py
-      - main.py
-      - utils/model_download.py
+      - scripts/hosting_*.py
+      - scripts/test_hosting_release.py
+      - scripts/wait_hosting_release.py
       - .github/workflows/hosting-runtime.yml
-  workflow_dispatch: {}
 
 permissions:
   contents: read
   packages: write
+  statuses: read
 
 jobs:
   runtime:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Verify release coordination
+        run: python3 -m unittest discover -s scripts -p 'test_hosting_release.py' -v
       - uses: docker/setup-buildx-action@v3
       - uses: docker/login-action@v3
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - uses: docker/build-push-action@v6
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        id: build
         with:
           context: .
           file: k8s/local-hosting/Dockerfile
@@ -35,3 +40,29 @@ jobs:
           platforms: linux/amd64
           push: true
           tags: ghcr.io/opensensor/light-object-detect:hosting-${{ github.sha }}
+      - name: Record this build's platform image
+        if: steps.build.outcome == 'success'
+        env:
+          BUILD_DIGEST: ${{ steps.build.outputs.digest }}
+        run: python3 scripts/hosting_release_manifest.py
+      - uses: actions/upload-artifact@v4
+        if: steps.build.outcome == 'success'
+        with:
+          name: hosting-release-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: hosting-release.json
+          retention-days: 30
+          if-no-files-found: error
+  deploy:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    needs: runtime
+    concurrency:
+      group: detector-production-verification
+      cancel-in-progress: true
+    runs-on: ubuntu-latest
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v4
+      - name: Verify this build is deployed to production and recovery
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python3 scripts/wait_hosting_release.py
diff --git a/scripts/deploy/README.md b/scripts/deploy/README.md
new file mode 100644
index 0000000..083c122
--- /dev/null
+++ b/scripts/deploy/README.md
@@ -0,0 +1,29 @@
+# Automatic paired detector deployment
+
+Every main push builds the hosting image and publishes a receipt containing its
+source revision, workflow run and immutable amd64 image digest. The operator host
+polls for successful builds every minute using its existing GitHub and cluster
+access. CI waits for a status specific to that build and fails if deployment is
+not verified. Pull requests test coordination without deploying.
+
+The agent warms cloud, moves traffic there, updates cloud and local, updates the
+recovery controller's expected image, returns traffic locally, and verifies the
+cloud persistent model cache before scaling cloud back to zero. Checkpoints make
+interrupted handoffs resumable. A new build waits for an active handoff; an older
+build cannot replace a newer completed release. Operator holds are preserved.
+
+Install once on the existing operator host (no new credentials):
+
+```sh
+install -Dm755 scripts/deploy/detector-release-launch.sh ~/.local/lib/opensensor-hosting/detector-release-launch.sh
+install -Dm644 scripts/deploy/opensensor-detector-release.service ~/.config/systemd/user/opensensor-detector-release.service
+install -Dm644 scripts/deploy/opensensor-detector-release.timer ~/.config/systemd/user/opensensor-detector-release.timer
+systemctl --user daemon-reload
+systemctl --user enable --now opensensor-detector-release.timer
+```
+
+The launcher follows main. Receipts and resumable state are private files under
+`~/.config/opensensor-hosting/detector-release/`. The progress snapshot includes
+existing deployment configuration; keep that directory private. Review failures
+with `journalctl --user -u opensensor-detector-release.service`. The timer retries
+a failed handoff; do not delete pending state during a deployment.
diff --git a/scripts/deploy/detector-release-launch.sh b/scripts/deploy/detector-release-launch.sh
new file mode 100644
index 0000000..815338b
--- /dev/null
+++ b/scripts/deploy/detector-release-launch.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+release_checkout="$HOME/.cache/opensensor-hosting/detector-release"
+if [[ ! -d "$release_checkout/.git" ]]; then
+    git clone --branch main --single-branch https://github.com/opensensor/light-object-detect.git "$release_checkout"
+fi
+git -C "$release_checkout" fetch origin main
+git -C "$release_checkout" merge --ff-only origin/main
+agent="$release_checkout/scripts/hosting_release.py"
+[[ -f "$agent" ]] || exit 0
+exec /usr/bin/python3 "$agent" --apply
diff --git a/scripts/deploy/opensensor-detector-release.service b/scripts/deploy/opensensor-detector-release.service
new file mode 100644
index 0000000..d9cb79b
--- /dev/null
+++ b/scripts/deploy/opensensor-detector-release.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Deploy latest main detector build to production and recovery
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+Environment=PATH=/snap/bin:/usr/local/bin:/usr/bin:/bin
+ExecStart=/usr/bin/bash %h/.local/lib/opensensor-hosting/detector-release-launch.sh
+TimeoutStartSec=35min
+UMask=0077
+Nice=5
diff --git a/scripts/deploy/opensensor-detector-release.timer b/scripts/deploy/opensensor-detector-release.timer
new file mode 100644
index 0000000..467b23f
--- /dev/null
+++ b/scripts/deploy/opensensor-detector-release.timer
@@ -0,0 +1,11 @@
+[Unit]
+Description=Watch for new main detector images
+
+[Timer]
+OnBootSec=30s
+OnUnitInactiveSec=1min
+AccuracySec=5s
+Unit=opensensor-detector-release.service
+
+[Install]
+WantedBy=timers.target
diff --git a/scripts/hosting_release.py b/scripts/hosting_release.py
new file mode 100644
index 0000000..06c13aa
--- /dev/null
+++ b/scripts/hosting_release.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""Consume the latest successful main image build and resume paired deployments."""
+import argparse
+import fcntl
+import io
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import time
+import zipfile
+
+from hosting_runtime import deploy, save_progress
+
+REPO = 'opensensor/light-object-detect'
+WORKFLOW = '.github/workflows/hosting-runtime.yml'
+CONTEXT = 'hosting/detector-release'
+STATE = Path.home() / '.config/opensensor-hosting/detector-release'
+IMAGE = re.compile(r'ghcr\.io/opensensor/light-object-detect@sha256:[a-f0-9]{64}')
+
+
+def gh(path, **fields):
+    args = ['gh', 'api', '-H', 'Cache-Control: no-cache', path]
+    for key, value in fields.items():args += ['-f', key + '=' + str(value)]
+    return json.loads(subprocess.check_output(args, timeout=90))
+
+
+def validate(target, workflow):
+    if (workflow['event'] != 'push' or workflow['head_branch'] != 'main'
+            or workflow['path'] != WORKFLOW or workflow['repository']['full_name'] != REPO
+            or set(target) != {'format', 'repository', 'commit', 'run_id', 'run_attempt', 'image'}
+            or target['format'] != 'detector-release-v1' or target['repository'] != REPO
+            or target['commit'] != workflow['head_sha'] or target['run_id'] != workflow['id']
+            or target['run_attempt'] != workflow['run_attempt']
+            or not re.fullmatch('[a-f0-9]{40}', target['commit']) or not IMAGE.fullmatch(target['image'])):
+        raise ValueError('Release must identify this main build and its immutable image')
+    return target
+
+
+def latest():
+    runs = gh(f'repos/{REPO}/actions/workflows/hosting-runtime.yml/runs?branch=main&event=push&per_page=20')['workflow_runs']
+    for workflow in sorted(runs, key=lambda r:(r['run_number'], r['run_attempt']), reverse=True):
+        jobs = gh(f'repos/{REPO}/actions/runs/{workflow["id"]}/attempts/{workflow["run_attempt"]}/jobs?per_page=100')['jobs']
+        if not any(j['name']=='runtime' and j['conclusion']=='success' for j in jobs):continue
+        artifacts = gh(f'repos/{REPO}/actions/runs/{workflow["id"]}/artifacts?per_page=100')['artifacts']
+        name = f'hosting-release-{workflow["head_sha"]}-{workflow["id"]}-{workflow["run_attempt"]}'
+        matches = [a for a in artifacts if a['name']==name and not a['expired']]
+        if len(matches)!=1:raise RuntimeError('Successful build is missing its release artifact')
+        raw = subprocess.check_output(['gh','api',f'repos/{REPO}/actions/artifacts/{matches[0]["id"]}/zip'],timeout=90)
+        with zipfile.ZipFile(io.BytesIO(raw)) as archive:
+            if archive.namelist()!=['hosting-release.json'] or archive.infolist()[0].file_size>8192:
+                raise ValueError('Unexpected release artifact')
+            return validate(json.loads(archive.read('hosting-release.json')),workflow)
+    return None
+
+
+def report(target, state, description):
+    gh(f'repos/{REPO}/statuses/{target["commit"]}',state=state,
+       context=f'{CONTEXT}/{target["run_id"]}/{target["run_attempt"]}',description=description[:140],
+       target_url=f'https://github.com/{REPO}/actions/runs/{target["run_id"]}')
+
+
+def choose(pending, candidate, previous):
+    # Finish a handoff already in progress before starting another release.
+    if pending:return pending
+    if not candidate:return None
+    if previous:
+        before=(previous['run_id'],previous['run_attempt'])
+        after=(candidate['run_id'],candidate['run_attempt'])
+        if after<before:raise RuntimeError('Refuse an older release')
+        if after==before and previous.get('state')=='success':return None
+    return candidate
+
+
+def main():
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--apply',action='store_true')
+    args=parser.parse_args()
+    os.umask(0o077);STATE.mkdir(parents=True,exist_ok=True)
+    with (STATE/'agent.lock').open('a') as lock:
+        fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+        read=lambda name:json.loads((STATE/name).read_text()) if (STATE/name).exists() else {}
+        pending, previous=read('pending.json'),read('latest.json')
+        target=choose(pending,None if pending else latest(),previous)
+        if not target:return
+        if not args.apply:print(json.dumps(target));return
+        save_progress(STATE/'pending.json',target)
+        directory=STATE/f'{target["run_id"]}-{target["run_attempt"]}'
+        directory.mkdir(exist_ok=True)
+        report(target,'pending','Automatically deploying this build to both detector origins')
+        try:
+            result=deploy(target,directory)
+            receipt={**target,**result,'state':'success','updated_at':time.time()}
+            report(target,'success','Exact image serving locally; cloud recovery image and persistent cache verified')
+            save_progress(STATE/'latest.json',receipt)
+            (STATE/'pending.json').unlink()
+        except Exception as error:
+            save_progress(STATE/'latest.json',{**target,'state':'failure','error_type':type(error).__name__,'updated_at':time.time()})
+            report(target,'failure','Automatic deployment will retry: '+type(error).__name__)
+            raise
+
+
+if __name__=='__main__':main()
diff --git a/scripts/hosting_release_manifest.py b/scripts/hosting_release_manifest.py
new file mode 100644
index 0000000..3b3f0da
--- /dev/null
+++ b/scripts/hosting_release_manifest.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""Bind a release to this build's amd64 manifest, excluding provenance attestations."""
+import json
+import os
+import subprocess
+
+
+def platform_digest(document, digest):
+    if 'manifests' not in document:return digest
+    matches=[m['digest'] for m in document['manifests'] if m.get('platform')=={'architecture':'amd64','os':'linux'}]
+    if len(matches)!=1:raise ValueError('Expected exactly one linux/amd64 image')
+    return matches[0]
+
+
+def main():
+    digest=os.environ['BUILD_DIGEST']
+    repository='ghcr.io/opensensor/light-object-detect'
+    document=json.loads(subprocess.check_output(['docker','buildx','imagetools','inspect',repository+'@'+digest,'--raw']))
+    target={'format':'detector-release-v1','repository':os.environ['GITHUB_REPOSITORY'],
+        'commit':os.environ['GITHUB_SHA'],'run_id':int(os.environ['GITHUB_RUN_ID']),
+        'run_attempt':int(os.environ['GITHUB_RUN_ATTEMPT']),
+        'image':repository+'@'+platform_digest(document,digest)}
+    with open('hosting-release.json','w') as out:json.dump(target,out)
+
+
+if __name__=='__main__':main()
diff --git a/scripts/hosting_runtime.py b/scripts/hosting_runtime.py
new file mode 100644
index 0000000..d638dff
--- /dev/null
+++ b/scripts/hosting_runtime.py
@@ -0,0 +1,176 @@
+"""Paired release with a warmed cloud origin and separate temporary model cache."""
+import copy
+import json
+from pathlib import Path
+import subprocess
+import time
+import urllib.request
+
+import shlex
+
+NS = 'light-object-detect'
+CLOUD = 'do-nyc1-k8s-1-33-1-do-1-nyc1-1752275240114'
+
+def command(site, args):
+    if site == 'cloud':
+        return ['kubectl','--context',CLOUD,'--request-timeout=20s',*args]
+    remote = ['/srv/ai/opensensor-hosting/home/bin/kubectl','--kubeconfig',
+              '/srv/ai/opensensor-hosting/config/kubeconfig','--context','kind-opensensor-local',
+              '--request-timeout=20s',*args]
+    return ['ssh','-o','BatchMode=yes','-o','ConnectTimeout=8',
+            'opensensor-hosting@192.168.53.187',shlex.join(remote)]
+
+def get(site, *args):
+    return json.loads(run(site,[*args,'-o','json']))
+
+
+def run(site, args, body=None):
+    result = subprocess.run(command(site, args), input=body, text=True, capture_output=True, timeout=620)
+    if result.returncode:raise RuntimeError('Cluster operation failed: ' + result.stderr[:400])
+    return result.stdout
+
+
+def patch(site, kind, name, operations):
+    obj = get(site, 'get', kind, name, '-n', NS)
+    return run(site, ['patch', kind, name, '-n', NS, '--type=json', '--patch-file=/dev/stdin'],
+               json.dumps([{'op': 'test', 'path': '/metadata/resourceVersion', 'value': obj['metadata']['resourceVersion']}] + operations))
+
+
+def placement():return get('cloud', 'get', 'cm', 'object-detection-placement', '-n', NS)
+
+
+def owned(revision):
+    data = placement()['data']
+    if data.get('revision') != revision or data.get('release_hold') != revision or data.get('desired') != 'cloud':
+        raise RuntimeError('Detector release hold changed')
+
+
+def wait(label, function, seconds=600):
+    deadline = time.monotonic() + seconds
+    while time.monotonic() < deadline:
+        if function():print(label, flush=True);return
+        time.sleep(3)
+    raise RuntimeError('Timed out: ' + label)
+
+
+def ready(site, image):
+    d = get(site, 'get', 'deploy', NS, '-n', NS)
+    status = d['status']
+    if d['spec']['replicas'] != 1 or d['spec']['template']['spec']['containers'][0]['image'] != image:return False
+    if status.get('observedGeneration', 0) < d['metadata']['generation']:return False
+    if any(status.get(k, 0) != 1 for k in ('replicas', 'updatedReplicas', 'readyReplicas')):return False
+    labels = ','.join(k + '=' + v for k, v in d['spec']['selector']['matchLabels'].items())
+    pods = get(site, 'get', 'pods', '-n', NS, '-l', labels)['items']
+    if len(pods) != 1 or pods[0]['metadata'].get('deletionTimestamp'):return False
+    statuses = pods[0].get('status', {}).get('containerStatuses', [])
+    return (len(statuses) == 1 and statuses[0].get('ready') is True
+            and statuses[0].get('imageID', '').endswith(image.split('@')[1]))
+
+
+def public():
+    with urllib.request.urlopen('https://detect.lightnvr.com/__hosting/health', timeout=10) as response:
+        body = json.load(response)
+        assert response.status == 200 and body['status'] == 'ok'
+        return response.headers.get('X-OpenSensor-Location')
+
+
+def update(site, image, revision, temporary_cache=False):
+    owned(revision)
+    d = get(site, 'get', 'deploy', NS, '-n', NS)
+    spec = copy.deepcopy(d['spec']['template']['spec'])
+    c = spec['containers'][0]
+    if c['image'] != image:
+        c['image'] = image
+        c['env'] = [e for e in c.get('env', []) if e['name'] != 'SENTRY_RELEASE']
+        c['env'].append({'name':'SENTRY_RELEASE','value':'light-object-detect@'+revision.split(':')[-1]})
+        changes = []
+        if temporary_cache:
+            spec['volumes'] = [{'name':v['name'],'emptyDir':{'sizeLimit':'5Gi'}}
+                              if v['name']=='hf-cache' else v for v in spec['volumes']]
+            changes.append({'op':'replace','path':'/spec/strategy','value':{
+                'type':'RollingUpdate','rollingUpdate':{'maxUnavailable':0,'maxSurge':1}}})
+        changes.append({'op':'replace','path':'/spec/template/spec','value':spec})
+        patch('cloud' if site=='cloud' else 'local','deploy',NS,changes)
+    wait(site+' exact image and model Ready',lambda:ready(site,image))
+
+
+def deploy(target, directory):
+    image, commit = target['image'], target['commit']
+    revision = 'ci:'+str(target['run_id'])+':'+commit
+    path = directory/'progress.json'
+    if path.exists():state=json.loads(path.read_text())
+    else:
+        data=placement()['data']
+        if data.get('desired')!='local' or any(data.get(k) for k in ('hold','release_hold','maintenance_hold')):
+            raise RuntimeError('Existing maintenance or release intent must finish first')
+        baseline=get('cloud','get','deploy',NS,'-n',NS)
+        if baseline['spec']['strategy']!={'type':'Recreate'}:
+            raise RuntimeError('Unexpected baseline cloud cache strategy')
+        state={'target':target,'phase':0,'baseline':baseline,'placement_revision':data.get('revision','')}
+        save_progress(path,state)
+    if state['target']!=target:raise RuntimeError('Release state belongs to another build')
+    baseline=state['baseline']
+    def stage(number, action):
+        if state['phase'] < number:
+            action()
+            state['phase']=number;save_progress(path,state)
+    def intent(values):
+        current=placement()['data']
+        if (current.get('release_hold') not in ('',None,revision)
+                or current.get('revision','') not in (state['placement_revision'],revision)
+                or any(current.get(k) for k in ('hold','maintenance_hold'))):
+            raise RuntimeError('Another operator owns detector placement')
+        changes=[{'op':'test','path':'/data','value':current}]
+        changes += [{'op':'add','path':'/data/'+key,'value':str(value)}
+                    for key,value in {**values,'revision':revision}.items()]
+        run('cloud',['patch','cm','object-detection-placement','-n',NS,
+                     '--type=json','--patch-file=/dev/stdin'],json.dumps(changes))
+    def warm():
+        intent({'recovery_test_until':time.time()+1800})
+        current=get('cloud','get','deploy',NS,'-n',NS)['spec']['template']['spec']['containers'][0]['image']
+        wait('Cloud model warmed',lambda:ready('cloud',current))
+    def hold():
+        intent({'revision':revision,'release_hold':revision,'desired':'cloud'})
+        wait('Cloud selected',lambda:public()=='cloud',120)
+    def gateway():
+        owned(revision)
+        d=get('cloud','get','deploy','object-detection-gateway','-n',NS)
+        i=next(i for i,c in enumerate(d['spec']['template']['spec']['containers']) if c['name']=='controller')
+        env=d['spec']['template']['spec']['containers'][i].get('env',[])
+        if next((e.get('value') for e in env if e['name']=='EXPECTED_IMAGE'),None)!=image:
+            env=[e for e in env if e['name']!='EXPECTED_IMAGE']+[{'name':'EXPECTED_IMAGE','value':image}]
+            patch('cloud','deploy','object-detection-gateway',[{'op':'add','path':f'/spec/template/spec/containers/{i}/env','value':env}])
+        run('cloud',['rollout','status','deploy/object-detection-gateway','-n',NS,'--timeout=600s'])
+    def local():
+        intent({'release_hold':'','desired':'local','recovery_test_until':'0'})
+        wait('Local selected and cloud drained',lambda:public()=='local' and get('cloud','get','deploy',NS,'-n',NS)['spec']['replicas']==0)
+    def cache():
+        intent({})
+        d=get('cloud','get','deploy',NS,'-n',NS)
+        volumes=d['spec']['template']['spec']['volumes']
+        desired=baseline['spec']['template']['spec']['volumes']
+        if volumes!=desired or d['spec']['strategy']!=baseline['spec']['strategy']:
+            # Status heartbeats must not conflict with this spec-only CAS.
+            operations=[{'op':'test','path':'/spec/replicas','value':0},
+                {'op':'test','path':'/spec/template/spec/containers/0/image','value':image},
+                {'op':'test','path':'/spec/template/spec/volumes','value':volumes},
+                {'op':'replace','path':'/spec/strategy','value':baseline['spec']['strategy']},
+                {'op':'replace','path':'/spec/template/spec/volumes','value':desired}]
+            run('cloud',['patch','deploy',NS,'-n',NS,'--type=json','--patch-file=/dev/stdin'],json.dumps(operations))
+        intent({'recovery_test_until':time.time()+600})
+        wait('Persistent cloud model Ready',lambda:ready('cloud',image))
+        intent({'recovery_test_until':'0'})
+        wait('Cloud recovery idle',lambda:get('cloud','get','deploy',NS,'-n',NS)['spec']['replicas']==0)
+    stage(1,warm);stage(2,hold)
+    stage(3,lambda:update('cloud',image,revision,True))
+    stage(4,lambda:update('local',image,revision))
+    stage(5,gateway);stage(6,local);stage(7,cache)
+    if not ready('local',image) or public()!='local':raise RuntimeError('Latest local detector not serving')
+    d=get('cloud','get','deploy',NS,'-n',NS)
+    if d['spec']['template']['spec']['containers'][0]['image']!=image or d['spec']['replicas']!=0:
+        raise RuntimeError('Cloud recovery template differs from this build')
+    return {'local_ready':True,'cloud_replicas':0,'persistent_cache_verified':True}
+
+
+def save_progress(path,state):
+    temp=path.with_suffix('.tmp');temp.write_text(json.dumps(state));temp.replace(path)
diff --git a/scripts/test_hosting_release.py b/scripts/test_hosting_release.py
new file mode 100644
index 0000000..17a3a4d
--- /dev/null
+++ b/scripts/test_hosting_release.py
@@ -0,0 +1,86 @@
+import copy
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import hosting_release as agent
+import hosting_runtime as runtime
+from hosting_release_manifest import platform_digest
+
+IMAGE = 'ghcr.io/opensensor/light-object-detect@sha256:'+'a'*64
+TARGET = dict(format='detector-release-v1',repository=agent.REPO,commit='b'*40,
+              run_id=100,run_attempt=1,image=IMAGE)
+WORKFLOW = dict(event='push',head_branch='main',path=agent.WORKFLOW,
+                repository={'full_name':agent.REPO},head_sha='b'*40,id=100,run_attempt=1)
+
+
+class ReleaseTests(unittest.TestCase):
+    def test_artifact_must_belong_to_exact_main_run(self):
+        self.assertEqual(agent.validate(TARGET,WORKFLOW),TARGET)
+        for key,value in [('run_id',99),('run_attempt',2),('commit','c'*40),
+                          ('image','ghcr.io/opensensor/light-object-detect:latest')]:
+            with self.subTest(key=key),self.assertRaises(ValueError):
+                agent.validate({**TARGET,key:value},WORKFLOW)
+        for key,value in [('event','pull_request'),('head_branch','feature')]:
+            with self.subTest(key=key),self.assertRaises(ValueError):
+                agent.validate(TARGET,{**WORKFLOW,key:value})
+
+    def test_interrupted_handoff_finishes_before_next_release(self):
+        newer={**TARGET,'run_id':101}
+        self.assertEqual(agent.choose(TARGET,newer,{}),TARGET)
+        self.assertEqual(agent.choose({},newer,{**TARGET,'state':'success'}),newer)
+        self.assertIsNone(agent.choose({},TARGET,{**TARGET,'state':'success'}))
+        with self.assertRaises(RuntimeError):agent.choose({},TARGET,{**newer,'state':'success'})
+
+    def test_image_selection_excludes_attestation(self):
+        manifest={'manifests':[
+            {'digest':'sha256:attestation','platform':{'architecture':'unknown','os':'unknown'}},
+            {'digest':'sha256:runtime','platform':{'architecture':'amd64','os':'linux'}}]}
+        self.assertEqual(platform_digest(manifest,'sha256:index'),'sha256:runtime')
+        manifest['manifests'].append(manifest['manifests'][1])
+        with self.assertRaises(ValueError):platform_digest(manifest,'sha256:index')
+
+    def test_ready_rejects_old_binary_and_pending_or_terminating_pods(self):
+        deployment={'metadata':{'generation':3},'spec':{'replicas':1,
+            'template':{'spec':{'containers':[{'image':IMAGE}]}},
+            'selector':{'matchLabels':{'app':'detector'}}},
+            'status':{'observedGeneration':3,'replicas':1,'updatedReplicas':1,'readyReplicas':1}}
+        pod={'metadata':{},'status':{'containerStatuses':[{'ready':True,'imageID':'docker-pullable://'+IMAGE}]}}
+        def ready(pods):
+            with patch.object(runtime,'get',side_effect=[deployment,{'items':pods}]):
+                return runtime.ready('local',IMAGE)
+        self.assertTrue(ready([pod]))
+        old=copy.deepcopy(pod);old['status']['containerStatuses'][0]['imageID']='old'
+        self.assertFalse(ready([old]))
+        self.assertFalse(ready([{'metadata':{},'status':{}}]))
+        terminating=copy.deepcopy(pod);terminating['metadata']['deletionTimestamp']='now'
+        self.assertFalse(ready([pod,terminating]))
+
+    def test_resumed_release_does_not_overwrite_operator_hold(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path=Path(tmp)
+            runtime.save_progress(path/'progress.json',{'target':TARGET,'phase':5,
+                'baseline':{},'placement_revision':'previous'})
+            with patch.object(runtime,'placement',return_value={'data':{
+                    'revision':'operator','release_hold':'operator','desired':'cloud'}}), \
+                    patch.object(runtime,'run') as mutate:
+                with self.assertRaisesRegex(RuntimeError,'Another operator'):
+                    runtime.deploy(TARGET,path)
+                mutate.assert_not_called()
+
+    def test_completed_resume_checks_runtime_without_restarting_handoff(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path=Path(tmp)
+            runtime.save_progress(path/'progress.json',{'target':TARGET,'phase':7,
+                'baseline':{},'placement_revision':'previous'})
+            with patch.object(runtime,'ready',return_value=True), \
+                    patch.object(runtime,'public',return_value='local'), \
+                    patch.object(runtime,'get',return_value={'spec':{'replicas':0,
+                      'template':{'spec':{'containers':[{'image':IMAGE}]}}}}), \
+                    patch.object(runtime,'run') as mutate:
+                self.assertTrue(runtime.deploy(TARGET,path)['persistent_cache_verified'])
+                mutate.assert_not_called()
+
+
+if __name__=='__main__':unittest.main()
diff --git a/scripts/wait_hosting_release.py b/scripts/wait_hosting_release.py
new file mode 100644
index 0000000..d08c440
--- /dev/null
+++ b/scripts/wait_hosting_release.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""Keep CI pending until this build is verified on both production origins."""
+import json
+import os
+import subprocess
+import time
+
+
+def main():
+    repo, commit = os.environ['GITHUB_REPOSITORY'], os.environ['GITHUB_SHA']
+    context = f'hosting/detector-release/{os.environ["GITHUB_RUN_ID"]}/{os.environ["GITHUB_RUN_ATTEMPT"]}'
+    deadline = time.monotonic() + 2100
+    while time.monotonic() < deadline:
+        raw = subprocess.check_output(['gh','api','--paginate','--slurp','-H','Cache-Control: no-cache',
+            f'repos/{repo}/commits/{commit}/statuses?per_page=100'],timeout=60)
+        statuses = [s for page in json.loads(raw) for s in page if s['context']==context]
+        if statuses:
+            status = max(statuses,key=lambda s:s['id'])
+            print(status['state']+': '+status['description'],flush=True)
+            if status['state']=='success':return
+            # The host resumes failed handoffs automatically; keep this run open for its retry.
+        else:print('Waiting for the production release agent',flush=True)
+        time.sleep(20)
+    raise SystemExit('This build has not been verified in production; inspect hosting/detector-release')
+
+
+if __name__=='__main__':main()