From 3632158fec30dd06193a016c6d334447a74db238 Mon Sep 17 00:00:00 2001 From: Etan Joseph Heyman Date: Mon, 5 Oct 2026 06:01:30 +0300 Subject: [PATCH 1/5] fix(scrub): wait for the same-day verified backup Co-Authored-By: brainlayerCodex-2c941903 running gpt-6.1-sol --- docs/configuration.md | 5 ++ src/brainlayer/cli/__init__.py | 7 +++ src/brainlayer/scrub_at_rest.py | 30 +++++++++- tests/test_scrub_at_rest_command.py | 92 +++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 009b14be..8dd874de 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -188,6 +188,11 @@ if the current total differs. The guarded mode holds the maintenance lock, requires an active enrichment pause sentinel and a verified backup receipt for that DB no older than 24 hours, and reuses VACUUM's quiet-window (04:00–06:00 local), idle-queue and writer gates. +`--wait-for-backup-seconds N` (default 0) lets guarded live apply poll every 30 seconds +for a qualifying receipt before taking the lock or quiescing services. Dry runs and +offline applies ignore it. Waiting ends at N seconds or when only 60 seconds remain +in the quiet window, with `verified-backup-timeout`; the default still refuses with +`verified-backup-required`. Receipt freshness continues to use `attempted_at`. It quiesces the fleet/throughput/tier-0 watchdogs, health-check healer, BrainBar UI/daemon, hotlane, watcher, drain, index, tier-3 ingest, decay and enrichment using maintenance's service helpers. It checks that jobs remain unloaded, BrainBar processes are gone and no writable database diff --git a/src/brainlayer/cli/__init__.py b/src/brainlayer/cli/__init__.py index 5cb94394..369346d8 100644 --- a/src/brainlayer/cli/__init__.py +++ b/src/brainlayer/cli/__init__.py @@ -2883,6 +2883,12 @@ def scrub_at_rest_command( expect_rows: int | None = typer.Option( None, "--expect-rows", min=0, help="Required guarded apply total from the preceding dry run." ), + wait_for_backup_seconds: int = typer.Option( + 0, + "--wait-for-backup-seconds", + min=0, + help="Wait for a verified backup before guarded apply; dry runs ignore this option.", + ), ) -> None: """Redact provider credentials on a copy or a guarded runtime DB; print counts only.""" from ..scrub_at_rest import scrub_at_rest @@ -2895,6 +2901,7 @@ def scrub_at_rest_command( batch_size=batch_size, allow_live_db=allow_live_db, expect_rows=expect_rows, + wait_for_backup_seconds=wait_for_backup_seconds, ) except Exception as exc: typer.echo( diff --git a/src/brainlayer/scrub_at_rest.py b/src/brainlayer/scrub_at_rest.py index d18b8035..5bdd72b8 100644 --- a/src/brainlayer/scrub_at_rest.py +++ b/src/brainlayer/scrub_at_rest.py @@ -52,6 +52,7 @@ BRAINBAR_EXIT_TIMEOUT_SECONDS = 30.0 +BACKUP_WAIT_SAFE_MARGIN_SECONDS = 60.0 _QUIESCE_DETAILS = frozenset( {"quiesce-services", "brainbar-process-probe", "lsof-writers", "process:BrainBar", "process:BrainBarDaemon"} | { @@ -285,6 +286,30 @@ def _live_requirements(config): raise ScrubAtRestError("verified backup within 24 hours required", reason="verified-backup-required") +def _wait_for_verified_backup(path, timeout_seconds): + """Wait without holding a maintenance lock or stopping any writer services.""" + config = maintenance.MaintenanceConfig(db_path=path, backup_reuse_max_age_hours=24) + deadline = time.monotonic() + timeout_seconds + while True: + try: + _live_requirements(config) + return + except ScrubAtRestError as exc: + if exc.reason != "verified-backup-required": + raise + remaining = min( + deadline - time.monotonic(), + maintenance._remaining_quiet_window_seconds(config) - BACKUP_WAIT_SAFE_MARGIN_SECONDS, + ) + if remaining <= 0: + raise ScrubAtRestError("verified backup wait timed out", reason="verified-backup-timeout") + time.sleep(min(30.0, remaining)) + # The final poll may have produced a receipt; inspect it on the next loop. + # Preserve the window margin even when that final receipt is available. + if maintenance._remaining_quiet_window_seconds(config) <= BACKUP_WAIT_SAFE_MARGIN_SECONDS: + raise ScrubAtRestError("verified backup wait timed out", reason="verified-backup-timeout") + + def _check_no_brainbar_processes(): # An unregistered UI can open the daemon bundle even after its job is booted out. try: @@ -400,12 +425,13 @@ def scrub_at_rest( providers: str = "google_oauth", allow_live_db: bool = False, expect_rows: int | None = None, + wait_for_backup_seconds: int = 0, ) -> dict: """Read-only surveys need no opt-in; guarded applies require a current row total.""" failure = None try: with value_free_sqlite_logging(): - if providers not in PROVIDER_MODES or not 1 <= batch_size <= 1000: + if providers not in PROVIDER_MODES or not 1 <= batch_size <= 1000 or wait_for_backup_seconds < 0: raise ValueError("invalid scrub options") selected = PROVIDER_MODES[providers] try: @@ -422,6 +448,8 @@ def scrub_at_rest( if dry_run: with ReadonlyStore(path) as store: return _run(store, True, batch_size, selected) + if allow_live_db and wait_for_backup_seconds > 0: + _wait_for_verified_backup(path, wait_for_backup_seconds) with _maintenance_lock(path): path = assert_not_live_db(path, allow_live=allow_live_db) if allow_live_db: diff --git a/tests/test_scrub_at_rest_command.py b/tests/test_scrub_at_rest_command.py index b6a297cc..04a1fddf 100644 --- a/tests/test_scrub_at_rest_command.py +++ b/tests/test_scrub_at_rest_command.py @@ -1064,3 +1064,95 @@ def fail(*args, **kwargs): module.scrub_at_rest(db.db_path, allow_live_db=True, expect_rows=live_guard.total) assert error.value.detail == "state:com.etanhey.brainlayer-fleet-watchdog" assert "private-probe-value" not in str(error.value.__cause__) + + +@pytest.mark.parametrize("mode", ["google_oauth", "context7", "exa_labeled"]) +@pytest.mark.parametrize("outcome", ["appears", "final-poll", "timeout", "window", "zero", "dry-run"]) +def test_backup_wait_before_lock_and_quiesce(db, live_guard, monkeypatch, mode, outcome): + from contextlib import contextmanager + + from brainlayer import maintenance + from brainlayer import scrub_at_rest as module + + live_guard.backup.unlink() + monkeypatch.setenv("BRAINLAYER_MCP_SOCKET", str(db.db_path.parent / "absent.sock")) + monkeypatch.setenv("BRAINLAYER_FORBID_BRAINBAR_SOCKET", "1") + elapsed, sleeps, locks = [0.0], [], [] + factory = maintenance.MaintenanceConfig + + def config(**kwargs): + result = factory(**kwargs) + start = live_guard.now if outcome != "window" else live_guard.now.replace(hour=5, minute=58, second=59) + result.now_fn = lambda: start + dt.timedelta(seconds=elapsed[0]) + return result + + monkeypatch.setattr(maintenance, "MaintenanceConfig", config) + + def sleep(seconds): + assert not locks + assert not any(isinstance(e, tuple) for e in live_guard.events) + elapsed[0] += seconds + sleeps.append(seconds) + if len(sleeps) == {"appears": 2, "final-poll": 3}.get(outcome): + live_guard.backup.write_text(json.dumps(live_guard.receipt) + "\n") + + monkeypatch.setattr(module, "time", SimpleNamespace(monotonic=lambda: elapsed[0], sleep=sleep)) + lock = module._maintenance_lock + + @contextmanager + def tracked_lock(path): + locks.append(path) + with lock(path): + yield + + monkeypatch.setattr(module, "_maintenance_lock", tracked_lock) + total = sum(t["rows"] for t in module._run(db, True, 100, module.PROVIDER_MODES[mode])["tables"].values()) + flags = ["--dry-run"] if outcome == "dry-run" else ["--allow-live-db", "--expect-rows", str(total)] + result = CliRunner().invoke( + app, + [ + "scrub-at-rest", + "--db", + str(db.db_path), + "--providers", + mode, + "--wait-for-backup-seconds", + "0" if outcome == "zero" else "90", + *flags, + ], + ) + if outcome in {"appears", "final-poll", "dry-run"}: + assert result.exit_code == 0, result.output + assert len(sleeps) == {"appears": 2, "final-poll": 3, "dry-run": 0}[outcome] + assert len(locks) == (0 if outcome == "dry-run" else 1) + else: + assert result.exit_code == 1, result.output + assert json.loads(result.stdout)["reason"] == ( + "verified-backup-required" if outcome == "zero" else "verified-backup-timeout" + ) + if outcome != "zero": + assert not locks + assert not any(isinstance(e, tuple) for e in live_guard.events) + assert elapsed[0] == {"timeout": 90, "window": 1, "zero": 0}[outcome] + + +@pytest.mark.parametrize("guarded", [False, True]) +def test_backup_wait_does_not_sleep_for_offline_apply_or_existing_receipt(db, live_guard, monkeypatch, guarded): + from brainlayer import chunk_origin_wipe + from brainlayer import scrub_at_rest as module + + if not guarded: + live_guard.backup.unlink() + monkeypatch.setattr(chunk_origin_wipe, "_live_db_candidates", lambda: []) + monkeypatch.setattr( + module, + "time", + SimpleNamespace(monotonic=lambda: 0, sleep=lambda _: pytest.fail("unexpected backup wait")), + ) + result = module.scrub_at_rest( + db.db_path, + allow_live_db=guarded, + expect_rows=live_guard.total, + wait_for_backup_seconds=90, + ) + assert result["tables"]["chunks"]["rows"] == 1 From 50b8e548f0c7028aa26ac8baeb56a7a7b12d8477 Mon Sep 17 00:00:00 2001 From: Etan Joseph Heyman Date: Mon, 5 Oct 2026 06:37:43 +0300 Subject: [PATCH 2/5] fix(scrub): enforce wait deadline and validate row count early Co-Authored-By: brainlayerCodex-2c941903 running gpt-6.1-sol --- src/brainlayer/scrub_at_rest.py | 12 +++++++----- tests/test_scrub_at_rest_command.py | 19 ++++++++++++------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/brainlayer/scrub_at_rest.py b/src/brainlayer/scrub_at_rest.py index 5bdd72b8..1d53fb29 100644 --- a/src/brainlayer/scrub_at_rest.py +++ b/src/brainlayer/scrub_at_rest.py @@ -13,7 +13,7 @@ from .chunk_origin_wipe import assert_not_live_db from .chunk_write import canonical_content_hash from .dedupe import BUSY_RETRY_ATTEMPTS, _busy_retry_delay, compute_dedupe_fields -from .maintenance import _maintenance_lock +from .maintenance import _maintenance_lock, _remaining_quiet_window_seconds from .pipeline.secret_scrub import scrub_secrets from .runtime_store import ReadonlyStore, WriterRuntimeStore from .vector_store import value_free_sqlite_logging @@ -299,14 +299,14 @@ def _wait_for_verified_backup(path, timeout_seconds): raise remaining = min( deadline - time.monotonic(), - maintenance._remaining_quiet_window_seconds(config) - BACKUP_WAIT_SAFE_MARGIN_SECONDS, + _remaining_quiet_window_seconds(config) - BACKUP_WAIT_SAFE_MARGIN_SECONDS, ) if remaining <= 0: raise ScrubAtRestError("verified backup wait timed out", reason="verified-backup-timeout") time.sleep(min(30.0, remaining)) - # The final poll may have produced a receipt; inspect it on the next loop. - # Preserve the window margin even when that final receipt is available. - if maintenance._remaining_quiet_window_seconds(config) <= BACKUP_WAIT_SAFE_MARGIN_SECONDS: + # Inspect the final receipt only within the deadline and window reserve. + window_left = _remaining_quiet_window_seconds(config) - BACKUP_WAIT_SAFE_MARGIN_SECONDS + if time.monotonic() > deadline or window_left <= 0: raise ScrubAtRestError("verified backup wait timed out", reason="verified-backup-timeout") @@ -449,6 +449,8 @@ def scrub_at_rest( with ReadonlyStore(path) as store: return _run(store, True, batch_size, selected) if allow_live_db and wait_for_backup_seconds > 0: + if expect_rows is None or expect_rows < 0: + raise ScrubAtRestError("expected row count required", reason="expected-row-count-required") _wait_for_verified_backup(path, wait_for_backup_seconds) with _maintenance_lock(path): path = assert_not_live_db(path, allow_live=allow_live_db) diff --git a/tests/test_scrub_at_rest_command.py b/tests/test_scrub_at_rest_command.py index 04a1fddf..3388f6e8 100644 --- a/tests/test_scrub_at_rest_command.py +++ b/tests/test_scrub_at_rest_command.py @@ -1067,7 +1067,9 @@ def fail(*args, **kwargs): @pytest.mark.parametrize("mode", ["google_oauth", "context7", "exa_labeled"]) -@pytest.mark.parametrize("outcome", ["appears", "final-poll", "timeout", "window", "zero", "dry-run"]) +@pytest.mark.parametrize( + "outcome", ["appears", "final-poll", "oversleep", "timeout", "window", "zero", "dry-run", "missing-count"] +) def test_backup_wait_before_lock_and_quiesce(db, live_guard, monkeypatch, mode, outcome): from contextlib import contextmanager @@ -1091,9 +1093,9 @@ def config(**kwargs): def sleep(seconds): assert not locks assert not any(isinstance(e, tuple) for e in live_guard.events) - elapsed[0] += seconds + elapsed[0] += seconds + (1 if outcome == "oversleep" and len(sleeps) == 2 else 0) sleeps.append(seconds) - if len(sleeps) == {"appears": 2, "final-poll": 3}.get(outcome): + if len(sleeps) == {"appears": 2, "final-poll": 3, "oversleep": 3}.get(outcome): live_guard.backup.write_text(json.dumps(live_guard.receipt) + "\n") monkeypatch.setattr(module, "time", SimpleNamespace(monotonic=lambda: elapsed[0], sleep=sleep)) @@ -1108,6 +1110,8 @@ def tracked_lock(path): monkeypatch.setattr(module, "_maintenance_lock", tracked_lock) total = sum(t["rows"] for t in module._run(db, True, 100, module.PROVIDER_MODES[mode])["tables"].values()) flags = ["--dry-run"] if outcome == "dry-run" else ["--allow-live-db", "--expect-rows", str(total)] + if outcome == "missing-count": + flags = ["--allow-live-db"] result = CliRunner().invoke( app, [ @@ -1127,13 +1131,14 @@ def tracked_lock(path): assert len(locks) == (0 if outcome == "dry-run" else 1) else: assert result.exit_code == 1, result.output - assert json.loads(result.stdout)["reason"] == ( - "verified-backup-required" if outcome == "zero" else "verified-backup-timeout" - ) + expected = "verified-backup-required" if outcome == "zero" else "verified-backup-timeout" + if outcome == "missing-count": + expected = "expected-row-count-required" + assert json.loads(result.stdout)["reason"] == expected if outcome != "zero": assert not locks assert not any(isinstance(e, tuple) for e in live_guard.events) - assert elapsed[0] == {"timeout": 90, "window": 1, "zero": 0}[outcome] + assert elapsed[0] == {"timeout": 90, "oversleep": 91, "window": 1, "zero": 0, "missing-count": 0}[outcome] @pytest.mark.parametrize("guarded", [False, True]) From 9c5c6802c9965d2ef2fd0e5a0628533a988d066d Mon Sep 17 00:00:00 2001 From: Etan Joseph Heyman Date: Mon, 5 Oct 2026 07:38:12 +0300 Subject: [PATCH 3/5] fix(scrub): leave polling time before the backup deadline Co-Authored-By: brainlayerCodex-2c941903 running gpt-6.1-sol --- docs/configuration.md | 5 +++-- src/brainlayer/scrub_at_rest.py | 2 +- tests/test_scrub_at_rest_command.py | 13 +++++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8dd874de..76682c4f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -188,9 +188,10 @@ if the current total differs. The guarded mode holds the maintenance lock, requires an active enrichment pause sentinel and a verified backup receipt for that DB no older than 24 hours, and reuses VACUUM's quiet-window (04:00–06:00 local), idle-queue and writer gates. -`--wait-for-backup-seconds N` (default 0) lets guarded live apply poll every 30 seconds +`--wait-for-backup-seconds N` (default 0) lets guarded live apply poll at intervals of up to 30 seconds for a qualifying receipt before taking the lock or quiescing services. Dry runs and -offline applies ignore it. Waiting ends at N seconds or when only 60 seconds remain +offline applies ignore it. Poll intervals shorten near the deadline. Waiting ends +at N seconds or when only 60 seconds remain in the quiet window, with `verified-backup-timeout`; the default still refuses with `verified-backup-required`. Receipt freshness continues to use `attempted_at`. It quiesces the fleet/throughput/tier-0 watchdogs, health-check healer, BrainBar UI/daemon, diff --git a/src/brainlayer/scrub_at_rest.py b/src/brainlayer/scrub_at_rest.py index 1d53fb29..78c77880 100644 --- a/src/brainlayer/scrub_at_rest.py +++ b/src/brainlayer/scrub_at_rest.py @@ -303,7 +303,7 @@ def _wait_for_verified_backup(path, timeout_seconds): ) if remaining <= 0: raise ScrubAtRestError("verified backup wait timed out", reason="verified-backup-timeout") - time.sleep(min(30.0, remaining)) + time.sleep(min(30.0, remaining / 2)) # Inspect the final receipt only within the deadline and window reserve. window_left = _remaining_quiet_window_seconds(config) - BACKUP_WAIT_SAFE_MARGIN_SECONDS if time.monotonic() > deadline or window_left <= 0: diff --git a/tests/test_scrub_at_rest_command.py b/tests/test_scrub_at_rest_command.py index 3388f6e8..85071c01 100644 --- a/tests/test_scrub_at_rest_command.py +++ b/tests/test_scrub_at_rest_command.py @@ -1093,9 +1093,14 @@ def config(**kwargs): def sleep(seconds): assert not locks assert not any(isinstance(e, tuple) for e in live_guard.events) - elapsed[0] += seconds + (1 if outcome == "oversleep" and len(sleeps) == 2 else 0) + elapsed[0] += seconds + (0.01 if outcome == "final-poll" else 0) + if outcome == "oversleep" and len(sleeps) == 2: + elapsed[0] = 91 sleeps.append(seconds) - if len(sleeps) == {"appears": 2, "final-poll": 3, "oversleep": 3}.get(outcome): + arrival = ( + elapsed[0] >= 85 if outcome == "final-poll" else len(sleeps) == {"appears": 2, "oversleep": 3}.get(outcome) + ) + if arrival: live_guard.backup.write_text(json.dumps(live_guard.receipt) + "\n") monkeypatch.setattr(module, "time", SimpleNamespace(monotonic=lambda: elapsed[0], sleep=sleep)) @@ -1127,7 +1132,7 @@ def tracked_lock(path): ) if outcome in {"appears", "final-poll", "dry-run"}: assert result.exit_code == 0, result.output - assert len(sleeps) == {"appears": 2, "final-poll": 3, "dry-run": 0}[outcome] + assert len(sleeps) == {"appears": 2, "final-poll": 5, "dry-run": 0}[outcome] assert len(locks) == (0 if outcome == "dry-run" else 1) else: assert result.exit_code == 1, result.output @@ -1138,7 +1143,7 @@ def tracked_lock(path): if outcome != "zero": assert not locks assert not any(isinstance(e, tuple) for e in live_guard.events) - assert elapsed[0] == {"timeout": 90, "oversleep": 91, "window": 1, "zero": 0, "missing-count": 0}[outcome] + assert elapsed[0] == {"timeout": 90, "oversleep": 91, "window": 0.5, "zero": 0, "missing-count": 0}[outcome] @pytest.mark.parametrize("guarded", [False, True]) From 9ab89d288e17e48002277f554f07684b153b48c7 Mon Sep 17 00:00:00 2001 From: Etan Joseph Heyman Date: Mon, 5 Oct 2026 07:56:02 +0300 Subject: [PATCH 4/5] test(scrub): simplify backup wait scenario selection Co-Authored-By: brainlayerCodex-2c941903 running gpt-6.1-sol --- tests/test_scrub_at_rest_command.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_scrub_at_rest_command.py b/tests/test_scrub_at_rest_command.py index 85071c01..7fd05504 100644 --- a/tests/test_scrub_at_rest_command.py +++ b/tests/test_scrub_at_rest_command.py @@ -1084,7 +1084,7 @@ def test_backup_wait_before_lock_and_quiesce(db, live_guard, monkeypatch, mode, def config(**kwargs): result = factory(**kwargs) - start = live_guard.now if outcome != "window" else live_guard.now.replace(hour=5, minute=58, second=59) + start = {"window": live_guard.now.replace(hour=5, minute=58, second=59)}.get(outcome, live_guard.now) result.now_fn = lambda: start + dt.timedelta(seconds=elapsed[0]) return result @@ -1093,9 +1093,8 @@ def config(**kwargs): def sleep(seconds): assert not locks assert not any(isinstance(e, tuple) for e in live_guard.events) - elapsed[0] += seconds + (0.01 if outcome == "final-poll" else 0) - if outcome == "oversleep" and len(sleeps) == 2: - elapsed[0] = 91 + elapsed[0] += seconds + {"final-poll": 0.01}.get(outcome, 0) + elapsed[0] = {("oversleep", 2): 91}.get((outcome, len(sleeps)), elapsed[0]) sleeps.append(seconds) arrival = ( elapsed[0] >= 85 if outcome == "final-poll" else len(sleeps) == {"appears": 2, "oversleep": 3}.get(outcome) From 18372f5fba59bbdba7dd0a6cbf4f985fdac17333 Mon Sep 17 00:00:00 2001 From: Etan Joseph Heyman Date: Mon, 5 Oct 2026 08:40:50 +0300 Subject: [PATCH 5/5] fix(scrub): reserve twenty minutes after backup wait Co-Authored-By: brainlayerCodex-2c941903 running gpt-6.1-sol --- docs/configuration.md | 7 +++-- src/brainlayer/scrub_at_rest.py | 6 +++- tests/test_scrub_at_rest_command.py | 48 ++++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 76682c4f..ef4b299c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -191,9 +191,10 @@ reuses VACUUM's quiet-window (04:00–06:00 local), idle-queue and writer gates. `--wait-for-backup-seconds N` (default 0) lets guarded live apply poll at intervals of up to 30 seconds for a qualifying receipt before taking the lock or quiescing services. Dry runs and offline applies ignore it. Poll intervals shorten near the deadline. Waiting ends -at N seconds or when only 60 seconds remain -in the quiet window, with `verified-backup-timeout`; the default still refuses with -`verified-backup-required`. Receipt freshness continues to use `attempted_at`. +at N seconds or when only 20 minutes remain in the quiet window, reserving that +time for the guarded run. A refusal after sleeping reports `verified-backup-timeout`; +if no wait is possible before the first sleep, it reports `verified-backup-required`, +as does the default. Receipt freshness continues to use `attempted_at`. It quiesces the fleet/throughput/tier-0 watchdogs, health-check healer, BrainBar UI/daemon, hotlane, watcher, drain, index, tier-3 ingest, decay and enrichment using maintenance's service helpers. It checks that jobs remain unloaded, BrainBar processes are gone and no writable database diff --git a/src/brainlayer/scrub_at_rest.py b/src/brainlayer/scrub_at_rest.py index 78c77880..982feb01 100644 --- a/src/brainlayer/scrub_at_rest.py +++ b/src/brainlayer/scrub_at_rest.py @@ -52,7 +52,7 @@ BRAINBAR_EXIT_TIMEOUT_SECONDS = 30.0 -BACKUP_WAIT_SAFE_MARGIN_SECONDS = 60.0 +BACKUP_WAIT_SAFE_MARGIN_SECONDS = 20 * 60.0 _QUIESCE_DETAILS = frozenset( {"quiesce-services", "brainbar-process-probe", "lsof-writers", "process:BrainBar", "process:BrainBarDaemon"} | { @@ -290,6 +290,7 @@ def _wait_for_verified_backup(path, timeout_seconds): """Wait without holding a maintenance lock or stopping any writer services.""" config = maintenance.MaintenanceConfig(db_path=path, backup_reuse_max_age_hours=24) deadline = time.monotonic() + timeout_seconds + slept = False while True: try: _live_requirements(config) @@ -302,8 +303,11 @@ def _wait_for_verified_backup(path, timeout_seconds): _remaining_quiet_window_seconds(config) - BACKUP_WAIT_SAFE_MARGIN_SECONDS, ) if remaining <= 0: + if not slept: + raise ScrubAtRestError("verified backup within 24 hours required", reason="verified-backup-required") raise ScrubAtRestError("verified backup wait timed out", reason="verified-backup-timeout") time.sleep(min(30.0, remaining / 2)) + slept = True # Inspect the final receipt only within the deadline and window reserve. window_left = _remaining_quiet_window_seconds(config) - BACKUP_WAIT_SAFE_MARGIN_SECONDS if time.monotonic() > deadline or window_left <= 0: diff --git a/tests/test_scrub_at_rest_command.py b/tests/test_scrub_at_rest_command.py index 7fd05504..cf73490e 100644 --- a/tests/test_scrub_at_rest_command.py +++ b/tests/test_scrub_at_rest_command.py @@ -1084,7 +1084,7 @@ def test_backup_wait_before_lock_and_quiesce(db, live_guard, monkeypatch, mode, def config(**kwargs): result = factory(**kwargs) - start = {"window": live_guard.now.replace(hour=5, minute=58, second=59)}.get(outcome, live_guard.now) + start = {"window": live_guard.now.replace(hour=5, minute=39, second=59)}.get(outcome, live_guard.now) result.now_fn = lambda: start + dt.timedelta(seconds=elapsed[0]) return result @@ -1093,11 +1093,13 @@ def config(**kwargs): def sleep(seconds): assert not locks assert not any(isinstance(e, tuple) for e in live_guard.events) - elapsed[0] += seconds + {"final-poll": 0.01}.get(outcome, 0) + elapsed[0] += seconds + {"final-poll": 0.01, "window": 1.0}.get(outcome, 0) elapsed[0] = {("oversleep", 2): 91}.get((outcome, len(sleeps)), elapsed[0]) sleeps.append(seconds) arrival = ( - elapsed[0] >= 85 if outcome == "final-poll" else len(sleeps) == {"appears": 2, "oversleep": 3}.get(outcome) + elapsed[0] >= 85 + if outcome == "final-poll" + else len(sleeps) == {"appears": 2, "oversleep": 3, "window": 1}.get(outcome) ) if arrival: live_guard.backup.write_text(json.dumps(live_guard.receipt) + "\n") @@ -1142,7 +1144,7 @@ def tracked_lock(path): if outcome != "zero": assert not locks assert not any(isinstance(e, tuple) for e in live_guard.events) - assert elapsed[0] == {"timeout": 90, "oversleep": 91, "window": 0.5, "zero": 0, "missing-count": 0}[outcome] + assert elapsed[0] == {"timeout": 90, "oversleep": 91, "window": 1.5, "zero": 0, "missing-count": 0}[outcome] @pytest.mark.parametrize("guarded", [False, True]) @@ -1165,3 +1167,41 @@ def test_backup_wait_does_not_sleep_for_offline_apply_or_existing_receipt(db, li wait_for_backup_seconds=90, ) assert result["tables"]["chunks"]["rows"] == 1 + + +@pytest.mark.parametrize("state", ["before-window", "after-window", "within-reserve", "missing-pause"]) +def test_backup_wait_refuses_without_sleep_for_unavailable_window_or_other_gate(db, live_guard, monkeypatch, state): + from brainlayer import maintenance + from brainlayer import scrub_at_rest as module + + live_guard.backup.unlink() + if state == "missing-pause": + live_guard.pause.unlink() + monkeypatch.setenv("BRAINLAYER_MCP_SOCKET", str(db.db_path.parent / "absent.sock")) + monkeypatch.setenv("BRAINLAYER_FORBID_BRAINBAR_SOCKET", "1") + start = { + "before-window": live_guard.now.replace(hour=3, minute=50), + "after-window": live_guard.now.replace(hour=7, minute=50), + "within-reserve": live_guard.now.replace(hour=5, minute=50), + }.get(state, live_guard.now) + factory = maintenance.MaintenanceConfig + elapsed, sleeps = [0.0], [] + + def config(**kwargs): + result = factory(**kwargs) + result.now_fn = lambda: start + dt.timedelta(seconds=elapsed[0]) + return result + + def sleep(seconds): + sleeps.append(seconds) + elapsed[0] += 1801 + + monkeypatch.setattr(maintenance, "MaintenanceConfig", config) + monkeypatch.setattr(module, "time", SimpleNamespace(monotonic=lambda: elapsed[0], sleep=sleep)) + monkeypatch.setattr(module, "_maintenance_lock", lambda _: pytest.fail("lock taken before refusal")) + reason = "enrichment-pause-required" if state == "missing-pause" else "verified-backup-required" + with pytest.raises(module.ScrubAtRestError) as error: + module.scrub_at_rest(db.db_path, allow_live_db=True, expect_rows=live_guard.total, wait_for_backup_seconds=1800) + assert error.value.reason == reason + assert sleeps == [] + assert not any(isinstance(e, tuple) for e in live_guard.events)