diff --git a/scripts/hosting_runtime.py b/scripts/hosting_runtime.py index d638dff..873b845 100644 --- a/scripts/hosting_runtime.py +++ b/scripts/hosting_runtime.py @@ -5,6 +5,7 @@ import subprocess import time import urllib.request +import urllib.error import shlex @@ -48,7 +49,10 @@ def owned(revision): def wait(label, function, seconds=600): deadline = time.monotonic() + seconds while time.monotonic() < deadline: - if function():print(label, flush=True);return + try: + if function():print(label, flush=True);return + except (urllib.error.URLError, TimeoutError): + pass # Health probes can fail while gateway endpoints are changing. time.sleep(3) raise RuntimeError('Timed out: ' + label) @@ -122,7 +126,7 @@ def intent(values): raise RuntimeError('Another operator owns detector placement') changes=[{'op':'test','path':'/data','value':current}] changes += [{'op':'add','path':'/data/'+key,'value':str(value)} - for key,value in {**values,'revision':revision}.items()] + for key,value in values.items()] run('cloud',['patch','cm','object-detection-placement','-n',NS, '--type=json','--patch-file=/dev/stdin'],json.dumps(changes)) def warm(): diff --git a/scripts/test_hosting_release.py b/scripts/test_hosting_release.py index 17a3a4d..374cf14 100644 --- a/scripts/test_hosting_release.py +++ b/scripts/test_hosting_release.py @@ -1,6 +1,7 @@ import copy import tempfile import unittest +import urllib.error from pathlib import Path from unittest.mock import patch @@ -16,6 +17,14 @@ class ReleaseTests(unittest.TestCase): + def test_health_probe_retries_transient_502_but_not_configuration_errors(self): + with patch.object(runtime.time,'sleep'), patch.object(runtime.time,'monotonic',side_effect=[0,1,2]): + probe=unittest.mock.Mock(side_effect=[urllib.error.HTTPError('https://example.invalid',502,'gateway',{},None),True]) + runtime.wait('Recovered health probe',probe,10) + self.assertEqual(probe.call_count,2) + with self.assertRaises(ValueError): + runtime.wait('Invalid deployment',unittest.mock.Mock(side_effect=ValueError('configuration changed')),10) + def test_artifact_must_belong_to_exact_main_run(self): self.assertEqual(agent.validate(TARGET,WORKFLOW),TARGET) for key,value in [('run_id',99),('run_attempt',2),('commit','c'*40), @@ -57,6 +66,26 @@ def ready(pods): terminating=copy.deepcopy(pod);terminating['metadata']['deletionTimestamp']='now' self.assertFalse(ready([pod,terminating])) + def test_warmup_preserves_the_admitted_placement_revision(self): + class StopAfterWarmIntent(Exception):pass + data={'desired':'local','release_hold':'','revision':'admitted'} + def get(site,*args): + if args[1]=='cm':return {'data':dict(data)} + if args[2]=='forms-hosting-gateway': + return {'spec':{'template':{'spec':{'containers':[{'name':'controller','env':[{'name':'EXPECTED_IMAGE','value':IMAGE}]}]}}}} + return {'spec':{'strategy':{'type':'Recreate'},'template':{'spec':{'containers':[{'image':IMAGE}]}}}} + def run(site,args,body=None): + for operation in __import__('json').loads(body): + if operation['op']=='add':data[operation['path'].split('/')[-1]]=operation['value'] + return '' + with tempfile.TemporaryDirectory() as tmp, patch.object(runtime,'get',side_effect=get), \ + patch.object(runtime,'run',side_effect=run), \ + patch.object(runtime,'ready',side_effect=StopAfterWarmIntent): + with self.assertRaises(StopAfterWarmIntent):runtime.deploy(TARGET,Path(tmp)) + self.assertEqual(data['revision'],'admitted') + self.assertEqual(data['desired'],'local') + self.assertGreater(float(data['recovery_test_until']),0) + def test_resumed_release_does_not_overwrite_operator_hold(self): with tempfile.TemporaryDirectory() as tmp: path=Path(tmp)