From ae9936b7b4aeda1b9e05b834a2c1bbbb4aa97995 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Thu, 12 Nov 2026 17:08:24 +0100 Subject: [PATCH 01/13] Add the backup-chain operation guard for delete/merge/restore Backup deletion needs mutual exclusion between a delete, a merge and a restore on the same volume's backup chain. Add DBController.claim_backup_chain_op: one FDB transaction that reads every backup of a volume's chain and refuses if any sibling is already MERGING/DELETING or is named by an active restore task's chain, else flips the target(s) to the new status -- the status field itself is then the exclusion every later caller checks. Restore contention is tracked via a new JobSchedule index (restore_chain_s3_id) rather than a manual counter: it derives directly from each active FN_BACKUP_RESTORE task's own chain_ids, so entries appear and disappear with the task's own writes, nothing to leak or decrement by hand. Restore task creation now goes through DBController.create_backup_restore_task_if_chain_clear, which joins the same sibling check into the transaction that creates the task, closing the gap between validating a chain restorable and the task existing. add_backup_restore_task gains a leading source_lvol_id parameter; callers and the two tests asserting on its positional call signature are updated. Not yet wired into delete_backup/trigger_merge -- the guard exists but nothing calls it yet. --- .../controllers/backup/controller.py | 5 +- .../controllers/tasks_controller.py | 29 +++- simplyblock_core/db_controller.py | 143 ++++++++++++++++++ simplyblock_core/models/job_schedule.py | 16 ++ tests/integration/backup/test_legacy.py | 2 +- .../backup/test_restore_node_selection.py | 6 +- 6 files changed, 187 insertions(+), 14 deletions(-) diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py index b0a21610db..78a049dfa4 100644 --- a/simplyblock_core/controllers/backup/controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -927,10 +927,11 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, bdev_name = f"{lvol.lvs_name}/{lvol.lvol_bdev}" if not tasks_controller.add_backup_restore_task( - pool.cluster_id, lvol.node_id, backup_id, bdev_name, + backup.lvol_id, pool.cluster_id, lvol.node_id, backup_id, bdev_name, chain.s3_ids_newest_first(), lvol_id=lvol_id, s3_config=s3_config.model_dump(exclude_none=True) if s3_config is not None else None): - raise RuntimeError("Failed to create restore task") + raise PreconditionError( + "Failed to create restore task: the source chain is being merged or deleted") return lvol_id diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index e9c7dad063..1889894b43 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -1578,10 +1578,18 @@ def add_group_backup_commit_task(group_backup): ) -def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids, - lvol_id="", s3_config=None): +def add_backup_restore_task(source_lvol_id, cluster_id, node_id, backup_id, lvol_name, + chain_ids, lvol_id="", s3_config=None): """Create the task that restores an S3 backup chain into a new lvol. + Unlike every other task type, this does not go through ``_add_task``: its + creation has to be atomic with checking that no backup of the SOURCE + chain (``source_lvol_id`` -- the volume the backup was taken from, not + ``lvol_id``, the new volume being restored into) is already + ``MERGING``/``DELETING``, so a delete or merge cannot slip into the gap + between that check and the task actually existing. See + ``DBController.create_backup_restore_task_if_chain_clear``. + Args: s3_config: set when the backup lives in a bucket that is not the cluster's own, so the runner has to attach a device of its own to @@ -1589,13 +1597,12 @@ def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids and deletes it -- scrubbing this field -- once the restore reaches a terminal state. Unset, the restore reads through the device the node already has for the cluster's own bucket. + + Returns the task uuid, or ``False`` if the source chain is being merged + or deleted. """ - return _add_task( - JobSchedule.FN_BACKUP_RESTORE, - cluster_id, - node_id, - "", - max_retry=constants.BACKUP_MAX_RETRIES, + task, reason = db.create_backup_restore_task_if_chain_clear( + source_lvol_id, cluster_id, node_id, function_params={ "backup_id": backup_id, "lvol_name": lvol_name, @@ -1603,7 +1610,13 @@ def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids "chain_ids": chain_ids, "s3_config": s3_config, }, + max_retry=constants.BACKUP_MAX_RETRIES, ) + if task is None: + logger.info(f"Cannot create restore task: {reason}") + return False + tasks_events.task_create(task) + return task.uuid def add_backup_merge_task(cluster_id, node_id, keep_backup_id, old_backup_id): diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index d8a96c85b2..2c15cc29ef 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -5,6 +5,7 @@ import os.path import struct import time +import uuid from typing import Any, ClassVar import fdb @@ -1489,6 +1490,148 @@ def release_backup_chain_locks(self, snapshot_ids): transactional = fdb.transactional(DBController._release_backup_chain_locks_tx) transactional(self, self.kv_store, ordered_snapshot_ids) + # ---- Backup chain operation guard (Single FDB Transaction) ---- + # + # Mutual exclusion between a delete, a merge and a restore on the same + # volume's backup chain. Unlike acquire_backup_chain_locks above (which + # only protects the creation-time enqueue window), this guards the whole + # of a destructive operation -- but it does so WITHOUT holding anything + # open: the chain is bounded (constants.BACKUP_MAX_CHAIN_LENGTH), so + # reading every sibling and flipping the target(s) fits in one + # transaction, and FDB's serializable isolation is what makes two + # concurrent claims over the same chain mutually exclusive (see + # simplyblock_core/AGENTS.md: a critical section that fits in one + # transaction needs no separate lock, distributed or otherwise). The + # status this writes -- MERGING or DELETING -- IS the lock for everything + # after the claim: every other caller refuses to touch a chain holding + # either, which is what makes a long-running merge/delete task safe + # without this transaction, or anything else, staying open for its + # duration. + + def _claim_backup_chain_op_tx(self, tr, lvol_id, target_ids, new_status): + idx = indices.get_index(Backup, 'lvol_id') + sibling_ids = [ + idx.entry_id(Backup, bytes(key), bytes(value)) + for key, value in tr.get_range_startswith(idx.prefix(Backup, (lvol_id,))) + ] + siblings = {} + for sibling_id in sibling_ids: + rows = Backup().read_from_db(tr, id=sibling_id) + if rows: + siblings[sibling_id] = rows[0] + + blocking = next((b for b in siblings.values() + if b.status in (Backup.STATUS_MERGING, Backup.STATUS_DELETING)), None) + if blocking is not None: + return False, f"backup {blocking.uuid} of this chain is already {blocking.status}" + + restore_idx = indices.get_index(JobSchedule, 'restore_chain_s3_id') + for backup in siblings.values(): + restoring = any(True for _ in tr.get_range_startswith( + restore_idx.prefix(JobSchedule, (backup.s3_id,)), limit=1)) + if restoring: + return False, f"backup {backup.uuid} is being read by an active restore" + + missing = [t for t in target_ids if t not in siblings] + if missing: + return False, (f"backup(s) {', '.join(missing)} not found in volume " + f"{lvol_id}'s chain") + + index_list = Backup.active_indexes(tr) + for target_id in target_ids: + target = siblings[target_id] + old_keys = BaseModel.index_keys(Backup, index_list, target) + target.status = new_status + target.updated_at = str(datetime.datetime.now(datetime.UTC)) + if index_list: + BaseModel._apply_index_diff(tr, Backup, index_list, old_keys, target) + tr[target.get_db_id().encode()] = json.dumps(target.to_dict(unwrap_secrets=True)).encode() + scope = target.watch_scope() + tr.add(watches.watch_index_rollup_key(Backup, scope), watches.ONE_LE64) + tr.add(watches.watch_index_version_key(Backup, scope, target.get_id()), watches.ONE_LE64) + + return True, None + + def claim_backup_chain_op(self, lvol_id, target_ids, new_status): + """Atomically claim one or more backups of ``lvol_id``'s chain for a + destructive operation (merge or delete). + + Refuses, as ``(False, reason)``, if any backup anywhere in the chain -- + not only the targets -- is already ``MERGING``/``DELETING``, or is named + by an active restore's chain (``JobSchedule.restore_chain_s3_id``). On + success, every id in ``target_ids`` is flipped to ``new_status`` and + ``(True, None)`` is returned. + + Assumes ``Backup``'s ``lvol_id`` index is ready, which it already is in + any deployment old enough to run this: an index still ``building`` would + make this read incomplete rather than merely slow, so this does not fall + back to a scan the way a read through ``query()`` would. + """ + if not self.kv_store or not target_ids: + return False, "No DB connection" if not self.kv_store else "No targets given" + transactional = fdb.transactional(DBController._claim_backup_chain_op_tx) + return transactional(self, self.kv_store, lvol_id, list(target_ids), new_status) + + def _create_backup_restore_task_tx(self, tr, source_lvol_id, cluster_id, node_id, + function_params, max_retry, task_uuid, task_date): + idx = indices.get_index(Backup, 'lvol_id') + sibling_ids = [ + idx.entry_id(Backup, bytes(key), bytes(value)) + for key, value in tr.get_range_startswith(idx.prefix(Backup, (source_lvol_id,))) + ] + for sibling_id in sibling_ids: + rows = Backup().read_from_db(tr, id=sibling_id) + if rows and rows[0].status in (Backup.STATUS_MERGING, Backup.STATUS_DELETING): + return None, f"backup {rows[0].uuid} of this chain is {rows[0].status}" + + task_obj = JobSchedule() + task_obj.uuid = task_uuid + task_obj.cluster_id = cluster_id + task_obj.node_id = node_id + task_obj.date = task_date + task_obj.function_name = JobSchedule.FN_BACKUP_RESTORE + task_obj.function_params = function_params + task_obj.max_retry = max_retry + task_obj.status = JobSchedule.STATUS_NEW + + index_list = JobSchedule.active_indexes(tr) + if index_list: + BaseModel._apply_index_diff( + tr, JobSchedule, index_list, BaseModel.index_keys(JobSchedule, index_list, None), + task_obj) + tr[task_obj.get_db_id().encode()] = json.dumps(task_obj.to_dict(unwrap_secrets=True)).encode() + scope = task_obj.watch_scope() + tr.add(watches.watch_index_rollup_key(JobSchedule, scope), watches.ONE_LE64) + tr.add(watches.watch_index_version_key(JobSchedule, scope, task_obj.get_id()), watches.ONE_LE64) + + return task_obj, None + + def create_backup_restore_task_if_chain_clear(self, source_lvol_id, cluster_id, node_id, + function_params, max_retry): + """Create the FN_BACKUP_RESTORE task that drives a restore, atomically + with checking that no backup of ``source_lvol_id``'s chain is already + ``MERGING``/``DELETING``. + + Joining the check and the task's creation in one transaction is what + closes the gap a plain ``chain.require_restorable()`` followed by a + separate ``write_to_db()`` would leave open: without it, a delete or + merge could claim the chain (``claim_backup_chain_op``) in between, + and this task would start reading objects that operation is about to + remove. The task itself, once created, is what ``claim_backup_chain_op`` + then sees (via the ``restore_chain_s3_id`` index) and refuses to let a + later delete or merge touch -- see that method and the index's + declaration on ``JobSchedule``. + + Returns ``(task, None)`` on success, or ``(None, reason)`` if refused. + """ + if not self.kv_store: + return None, "No DB connection" + task_uuid = str(uuid.uuid4()) + task_date = int(time.time()) + transactional = fdb.transactional(DBController._create_backup_restore_task_tx) + return transactional(self, self.kv_store, source_lvol_id, cluster_id, node_id, + function_params, max_retry, task_uuid, task_date) + # ---- Per-lvstore snapshot-mutation lock (Single FDB Transaction) ---- def _try_acquire_lvstore_lock_tx(self, tr, cluster_id, lvs_name, owner, now, ttl): diff --git a/simplyblock_core/models/job_schedule.py b/simplyblock_core/models/job_schedule.py index 31d729421d..6d6ec7cee7 100644 --- a/simplyblock_core/models/job_schedule.py +++ b/simplyblock_core/models/job_schedule.py @@ -35,6 +35,22 @@ class JobSchedule(BaseModel): and t.function_params.get('snapshot_id') else [] )), + # Every backup s3_id a non-terminal FN_BACKUP_RESTORE task is reading + # (its chain_ids -- see FN_BACKUP_RESTORE below -- are s3_ids, not + # backup uuids). db_controller.claim_backup_chain_op reads this before + # letting a delete or merge touch a chain, so a restore in flight blocks + # them. Self-maintaining: the extractor yields nothing once the task + # reaches STATUS_DONE or is canceled, so the entry disappears in the + # same transaction as that write -- nothing decrements it by hand, and + # nothing can leak it. + Index('restore_chain_s3_id', arity=1, extract=lambda t: ( + [(s3_id,) for s3_id in (t.function_params.get('chain_ids') or [])] + if t.function_name == t.FN_BACKUP_RESTORE + and isinstance(t.function_params, dict) + and t.status != t.STATUS_DONE + and not t.canceled + else [] + )), ) STATUS_NEW = 'new' diff --git a/tests/integration/backup/test_legacy.py b/tests/integration/backup/test_legacy.py index a0e68f6883..9f3a15e208 100644 --- a/tests/integration/backup/test_legacy.py +++ b/tests/integration/backup/test_legacy.py @@ -708,7 +708,7 @@ def test_success(self, mock_tasks): self.assertEqual(result, "lvol-new") # s3_id integers reach the data plane, not backup UUIDs. - self.assertEqual(mock_tasks.add_backup_restore_task.call_args[0][4], [5]) + self.assertEqual(mock_tasks.add_backup_restore_task.call_args[0][5], [5]) self.assertEqual(self.db.get_lvol_by_id("lvol-new").status, LVol.STATUS_RESTORING) def test_backup_not_found(self): diff --git a/tests/unit/backup/test_restore_node_selection.py b/tests/unit/backup/test_restore_node_selection.py index c7d8298186..bb96c04fb6 100644 --- a/tests/unit/backup/test_restore_node_selection.py +++ b/tests/unit/backup/test_restore_node_selection.py @@ -116,7 +116,7 @@ def test_restore_task_targets_the_node_the_volume_landed_on(self, db, add_lvol_h _restore() - cluster_id, node_id, *_ = tasks.add_backup_restore_task.call_args.args + source_lvol_id, cluster_id, node_id, *_ = tasks.add_backup_restore_task.call_args.args assert cluster_id == TARGET_CLUSTER assert node_id == "target-node" @@ -201,10 +201,10 @@ def test_volume_creation_failure_is_a_runtime_error(self, db, add_lvol_ha): with pytest.raises(RuntimeError, match="Failed to create restore volume"): _restore() - def test_task_creation_failure_is_a_runtime_error(self, db, add_lvol_ha, tasks): + def test_task_creation_failure_is_a_precondition_error(self, db, add_lvol_ha, tasks): tasks.add_backup_restore_task.return_value = False - with pytest.raises(RuntimeError, match="Failed to create restore task"): + with pytest.raises(PreconditionError, match="Failed to create restore task"): _restore() From ef27f64a185a2bb9d868072ece8b4ec6702a162d Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 13 Nov 2026 01:41:21 +0100 Subject: [PATCH 02/13] Implement backup deletion: manifest state, delete task, delete/merge rewrite Fixes a real bug found while wiring up claim_backup_chain_op: siblings were keyed by the composite get_id() but every caller holds a bare uuid, so the target-exemption and lookup checks never matched. Siblings are now keyed by uuid, and a target already in its own claimed state no longer blocks re-claiming itself (resuming an interrupted delete), while a genuinely different busy sibling still does. trigger_merge now goes through claim_backup_chain_op instead of writing MERGING directly, and returns (ok, reason) instead of silently doing nothing, so delete_backup can surface a real refusal to its own caller. manifest.py: BackupManifestState (ACTIVE/DELETING) and a state field on BackupManifest, mirroring GroupBackupManifest. mark_deleting/mark_active are idempotent read-modify-writes; controller.py wraps them as mark_manifest_deleting/revert_manifest_deleting. import_backups refuses a non-ACTIVE manifest for an explicit target; import_from_bucket skips one instead, logging it, the same split import_group_backup/ import_group_backup_from_bucket already use for group manifests. delete_backup/delete_backups rewritten around the guard: claim the chain, mark the manifest(s) DELETING, queue FN_BACKUP_DELETE -- reverting every claim just taken if marking a manifest fails, so a partial failure never leaves FDB and the bucket disagreeing. A backup with a completed successor is routed to trigger_merge instead (deleting it and merging it into that successor are the same request). Resuming an already-DELETING backup re-queues rather than re-claims. delete_backups additionally refuses while any backup of the volume is still being created or merging, and removes FAILED backups (no manifest, nothing to claim) directly. tasks_runner_backup.py: FN_BACKUP_DELETE handler (_run_delete) and its finalize_resource revert branch, following the existing FN_BACKUP_MERGE shape. Single-cycle: bdev_lvol_s3_delete has no paired stat RPC to poll, unlike backup/restore/merge. New unit tests for the manifest state functions (fake in-memory bucket, no DB). Integration test file for delete_backup/delete_backups/trigger_merge rewritten for the new async, claim-based behavior, including the new restore-vs-delete contention path -- NOT verified against real FDB in this sandbox; the testcontainers-provisioned FDB and a manually-configured single-process fallback both failed to come up usably here (environment limitation, not a code issue surfaced by either attempt). Lint, mypy, and the full unit suite (1778 tests) are green. --- .../controllers/backup/controller.py | 289 +++++++++++----- .../controllers/backup/manifest.py | 80 ++++- .../controllers/tasks_controller.py | 28 ++ simplyblock_core/db_controller.py | 46 ++- simplyblock_core/models/job_schedule.py | 5 + .../services/tasks_runner_backup.py | 90 ++++- .../integration/backup/test_delete_backup.py | 319 +++++++++++++++--- .../unit/backup/test_manifest_delete_state.py | 139 ++++++++ 8 files changed, 854 insertions(+), 142 deletions(-) create mode 100644 tests/unit/backup/test_manifest_delete_state.py diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py index 78a049dfa4..19506ec8b5 100644 --- a/simplyblock_core/controllers/backup/controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -30,7 +30,6 @@ from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.storage_node import StorageNode -from simplyblock_core.rpc_client import RPCException logger = logging.getLogger() @@ -378,6 +377,28 @@ def delete_manifest(backup: Backup) -> None: backup_manifest.delete(_config_for(backup), UUID(backup.uuid)) +def mark_manifest_deleting(backup: Backup) -> None: + """Mark a backup's manifest DELETING, before any of its S3 objects are + touched. See ``manifest.mark_deleting``. + + Raises: + ManifestError: the manifest could not be read or rewritten. + PreconditionError: the backup's bucket is not the cluster's own. + """ + backup_manifest.mark_deleting(_config_for(backup), UUID(backup.uuid)) + + +def revert_manifest_deleting(backup: Backup) -> None: + """Revert a backup's manifest to ACTIVE: a delete that did not finish. + See ``manifest.mark_active``. + + Raises: + ManifestError: the manifest could not be read or rewritten. + PreconditionError: the backup's bucket is not the cluster's own. + """ + backup_manifest.mark_active(_config_for(backup), UUID(backup.uuid)) + + def _blob_parent_id(snapshot: SnapShot) -> str | None: """The snapshot this one's blob is a delta against, or ``None`` at a root. @@ -936,7 +957,7 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, return lvol_id -def _cleanup_backup_kms_keys(backups): +def cleanup_backup_kms_keys(backups): encrypted = [b for b in backups if b.encrypted] if not encrypted: return @@ -953,15 +974,75 @@ def _cleanup_backup_kms_keys(backups): logger.exception("Failed to clean up backup KMS keys") +def _revert_backup_claim(backup_id: str) -> None: + """Undo a claim made by :func:`_start_backup_delete`/``trigger_merge`` that + did not get as far as anything irreversible: put the backup back to + COMPLETED. + + A no-op if the backup is gone (improbable here -- nothing between the + claim and this has removed it) or has moved past DELETING on its own. + """ + try: + fresh = db_controller.get_backup_by_id(backup_id) + except KeyError: + return + if fresh.status != Backup.STATUS_DELETING: + return + db_controller.atomic_update(fresh, lambda b: setattr(b, "status", Backup.STATUS_COMPLETED)) + + +def _start_backup_delete(backups: list[Backup]): + """Claim every backup in ``backups`` -- all COMPLETED, all belonging to one + volume's chain -- for deletion, mark their manifests DELETING, and queue + the task that removes their S3 objects. + + All-or-nothing: the chain claim (``DBController.claim_backup_chain_op``) + is one transaction already, and if marking any manifest afterwards fails, + every claim just taken is reverted back to COMPLETED before this returns + -- so a partial failure here never leaves a backup DELETING with an + ACTIVE manifest, or a manifest DELETING with no FDB record to match it + (the manifest write only happens after the claim, never before). + + Returns ``(True, None)`` once the delete task is queued (or an equivalent + one already was -- ``add_backup_delete_task`` dedupes), or + ``(False, reason)``. + """ + if not backups: + return True, None + lvol_id = backups[0].lvol_id + ids = [b.uuid for b in backups] + + ok, reason = db_controller.claim_backup_chain_op(lvol_id, ids, Backup.STATUS_DELETING) + if not ok: + return False, reason + + try: + for b in backups: + mark_manifest_deleting(b) + except (backup_manifest.ManifestError, PreconditionError, ValueError, KeyError) as e: + # KeyError: _config_for's own cluster lookup, inside + # mark_manifest_deleting. Everything here ran after the claim already + # committed, so every target is reverted -- there is nothing partial + # to distinguish. + for backup_id in ids: + _revert_backup_claim(backup_id) + return False, str(e) + + tasks_controller.add_backup_delete_task(backups[0].cluster_id, backups[0].node_id, ids) + return True, None + + def delete_backups(lvol_id): - """Delete all backups for a given lvol. + """Delete every backup of a volume. - Removes the database records, not the objects: bdev_lvol_s3_delete does not - exist on the data plane, so the S3 data outlives this call. The manifests - are deliberately left in place too -- they are the only thing that can still - identify those objects, and deleting them would turn a reclaimable orphan - set into anonymous bucket weight. `backup discover` therefore keeps showing - them, which is the honest answer about what the bucket contains. + All of them go directly to deletion, never a merge: nothing outside this + call survives to chain against any of them, so there is nothing to fold + one into. Refuses outright if any backup of the volume is still being + created (PENDING/IN_PROGRESS) -- deleting mid-upload would remove the very + backup a running upload's own ``prev_backup_id`` depends on -- or if any + is a live member of a group backup, or already MERGING (someone else's + merge of a backup this call would otherwise also claim). A backup already + DELETING is tolerated: this call resuming one that did not finish. Returns (success, error_message). """ @@ -969,65 +1050,66 @@ def delete_backups(lvol_id): if not backups: return False, f"No backups found for lvol {lvol_id}" - _cleanup_backup_kms_keys(backups) - - # Find node to run delete RPC on - completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] - if not completed: - # Just remove from DB - for b in backups: - b.remove(db_controller.kv_store) - return True, None + in_flight = [b for b in backups + if b.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS)] + if in_flight: + return False, (f"backup(s) {', '.join(b.uuid[:8] for b in in_flight)} of volume " + f"{lvol_id} are still being created") - node_id = completed[0].node_id - try: - snode = db_controller.get_storage_node_by_id(node_id) - except KeyError: - # Node gone, just clean up DB - for b in backups: - b.remove(db_controller.kv_store) - return True, None + merging = [b for b in backups if b.status == Backup.STATUS_MERGING] + if merging: + return False, (f"backup(s) {', '.join(b.uuid[:8] for b in merging)} of volume " + f"{lvol_id} are already being merged") - # Call S3 delete RPC (dummy for now) - if snode.status == StorageNode.STATUS_ONLINE: - rpc_client = snode.rpc_client() - s3_ids = [b.s3_id for b in completed] + for b in backups: + if not b.group_backup_id: + continue try: - rpc_client.bdev_lvol_s3_delete(s3_ids) - except Exception as e: - logger.error(f"Error deleting S3 backups: {e}") + owner = db_controller.get_group_backup_by_id(b.group_backup_id) + except KeyError: + continue # its group backup is gone: an orphaned member copy + if owner.status != GroupBackup.STATUS_DELETING: + return False, (f"backup {b.uuid} belongs to group backup " + f"{b.group_backup_id}, which still exists; delete the " + "group backup instead") - cluster_id = completed[0].cluster_id + completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] + if completed: + ok, reason = _start_backup_delete(completed) + if not ok: + return False, f"Cannot delete backups of volume {lvol_id}: {reason}" + + # A FAILED backup uploaded nothing restorable and has no manifest: remove + # its record directly. COMPLETED backups are left for the task just + # queued above to retire; a backup already DELETING is being retired by + # that task or an earlier one. for b in backups: - backup_events.backup_deleted(cluster_id, node_id, b) + if b.status != Backup.STATUS_FAILED: + continue + cleanup_backup_kms_keys([b]) + backup_events.backup_deleted(b.cluster_id, b.node_id, b) b.remove(db_controller.kv_store) return True, None def delete_backup(backup_id): - """Delete ONE backup: its record, its KMS keys and, best effort, its S3 - objects -- the single-record form of :func:`delete_backups`, for a backup - that nothing chains against any more, such as a deleted group backup's - member that is the last link of its volume's chain. + """Delete ONE backup: start deleting it if nothing chains against it, or + fold it into its completed successor if something does -- deleting a + backup with a successor and merging it into that successor are the same + request; see :func:`trigger_merge`. Raises: KeyError: no backup has this id. - PreconditionError: another backup names this one as its predecessor, - so deleting it would make every descendant unrestorable -- merge it - into its successor instead (``trigger_merge``), after which it is - MERGED, not deleted, and its objects are the survivor's; or it is a - member of a group backup that still exists, which deleting it would - tear. + PreconditionError: it is the predecessor of a backup that is not yet + completed (nothing to merge it into yet); it is a member of a + group backup that still exists (delete the group backup + instead); it is not completed itself; or the chain it belongs to + is already claimed by another merge, delete, or an active + restore (``DBController.claim_backup_chain_op``). """ backup = db_controller.get_backup_by_id(backup_id) - dependents = [b.uuid for b in db_controller.get_backups_by_lvol_id(backup.lvol_id) - if b.prev_backup_id == backup.uuid] - if dependents: - raise PreconditionError( - f"backup {backup.uuid} is the predecessor of " - f"{', '.join(d[:8] for d in dependents)}; merge it into its " - "successor instead of deleting it") + if backup.group_backup_id: try: owner = db_controller.get_group_backup_by_id(backup.group_backup_id) @@ -1039,21 +1121,34 @@ def delete_backup(backup_id): f"backup {backup.uuid} is a member of group backup " f"{backup.group_backup_id}; delete the group backup instead") - _cleanup_backup_kms_keys([backup]) - if backup.status == Backup.STATUS_COMPLETED: - try: - snode = db_controller.get_storage_node_by_id(backup.node_id) - except KeyError: - snode = None - if snode is not None and snode.status == StorageNode.STATUS_ONLINE: - try: - snode.rpc_client().bdev_lvol_s3_delete([backup.s3_id]) - except RPCException as e: - # Best effort, as delete_backups: the data plane does not - # implement the S3 delete yet, and the record goes regardless. - logger.error(f"Error deleting S3 objects of backup {backup.uuid}: {e}") - backup_events.backup_deleted(backup.cluster_id, backup.node_id, backup) - backup.remove(db_controller.kv_store) + if backup.status == Backup.STATUS_DELETING: + # Resuming a delete that claimed this backup but did not finish + # queuing its task (the process died in between): re-queue rather + # than re-claim, which would see this backup's own DELETING status + # among its siblings and refuse itself. + tasks_controller.add_backup_delete_task(backup.cluster_id, backup.node_id, [backup.uuid]) + return + + dependents = [b for b in db_controller.get_backups_by_lvol_id(backup.lvol_id) + if b.prev_backup_id == backup.uuid] + if dependents: + successor = next((b for b in dependents if b.status == Backup.STATUS_COMPLETED), None) + if successor is None: + raise PreconditionError( + f"backup {backup.uuid} is the predecessor of " + f"{', '.join(d.uuid[:8] for d in dependents)}, none of which is " + "completed yet; it cannot be merged or deleted") + ok, reason = trigger_merge(successor, backup) + if not ok: + raise PreconditionError(f"Cannot delete backup {backup.uuid}: {reason}") + return + + if backup.status != Backup.STATUS_COMPLETED: + raise PreconditionError(f"backup {backup.uuid} is {backup.status}, not completed") + + ok, reason = _start_backup_delete([backup]) + if not ok: + raise PreconditionError(f"Cannot delete backup {backup.uuid}: {reason}") def list_backups(cluster_id=None, show_all=False): @@ -1200,6 +1295,16 @@ def import_backups(export: backup_manifest.BackupExport, cluster_id=None) -> int for manifest in group.manifests: backup_id = manifest.backup_id + # A manifest a delete has marked DELETING describes objects that + # may already be gone -- the single-backup analog of + # import_group_backup's refusal of a non-COMMITTED group + # manifest. Explicit, named targets are refused outright here; + # import_from_bucket is where a bulk import skips these instead. + if manifest.state != backup_manifest.BackupManifestState.ACTIVE: + raise PreconditionError( + f"Backup {backup_id} is {manifest.state} in its bucket, " + f"not active; it cannot be imported") + if backup_id in pending: raise ValueError(f"Backup {backup_id} is listed more than once") @@ -1257,37 +1362,67 @@ def import_backups(export: backup_manifest.BackupExport, cluster_id=None) -> int def import_from_bucket(config: BackupConfig, cluster_id=None) -> int: """Import every backup found in a bucket. + A bucket holds backups being deleted on their source cluster too; they + are not part of what it can restore, so they are left, not refused -- + the single-backup analog of ``import_group_backup_from_bucket`` leaving a + non-``COMMITTED`` group manifest in place. A backup whose ancestor was + left this way is itself refused by the chain check in `import_backups`, + which is the correct outcome: its chain is incomplete in this bucket for + as long as the ancestor is being deleted. + Raises: ManifestError: the bucket could not be read. PreconditionError: the manifests it holds cannot be imported as a batch. """ + active = [] + for manifest in discover_backups(config): + if manifest.state != backup_manifest.BackupManifestState.ACTIVE: + logger.info("Backup %s is %s in the bucket; not importing it", + manifest.backup_id, manifest.state) + continue + active.append(manifest) + return import_backups( backup_manifest.BackupExport(groups=[ backup_manifest.LocatedManifests( location=config.location(), - manifests=discover_backups(config)), + manifests=active), ]), cluster_id=cluster_id) def trigger_merge(keep_backup, old_backup): """Merge ``old_backup`` into ``keep_backup``, its successor in one volume's - chain: mark the old one merging and queue the merge task. - - Both must be completed; otherwise nothing happens and a later pass retries. - Used by the single-volume retention (policy) and by the group backup's - retirement of a pruned generation's members (group_backup). + chain: claim the chain, mark the old one merging, and queue the merge task. + + Both must be completed, and the chain must not already be claimed by + another merge, delete, or an active restore + (``DBController.claim_backup_chain_op``) -- otherwise nothing happens and + a later pass retries. Used by the single-volume retention (policy), by + the group backup's retirement of a pruned generation's members + (group_backup), and by ``delete_backup`` when the backup requested for + deletion has a completed successor: deleting it and merging it into that + successor are the same operation up to this point, so both go through + here rather than duplicating the claim. + + Returns ``(True, None)`` once the merge task is queued, or + ``(False, reason)`` if nothing happened. The scheduled callers above + treat ``False`` as "try again next cycle" and ignore the reason; + ``delete_backup`` surfaces it to its own caller. """ if old_backup.status != Backup.STATUS_COMPLETED: - return + return False, f"backup {old_backup.uuid} is {old_backup.status}, not completed" if keep_backup.status != Backup.STATUS_COMPLETED: - return + return False, f"backup {keep_backup.uuid} is {keep_backup.status}, not completed" - old_backup.status = Backup.STATUS_MERGING - old_backup.write_to_db() + ok, reason = db_controller.claim_backup_chain_op( + old_backup.lvol_id, [old_backup.uuid], Backup.STATUS_MERGING) + if not ok: + return False, reason tasks_controller.add_backup_merge_task( keep_backup.cluster_id, keep_backup.node_id, keep_backup.uuid, old_backup.uuid) + return True, None diff --git a/simplyblock_core/controllers/backup/manifest.py b/simplyblock_core/controllers/backup/manifest.py index 0a2d6b6aa9..d118ff12ee 100644 --- a/simplyblock_core/controllers/backup/manifest.py +++ b/simplyblock_core/controllers/backup/manifest.py @@ -273,6 +273,25 @@ class ManifestDataPlane(BaseModel): with_compression: bool = False +class BackupManifestState(StrEnum): + """Whether a backup manifest's objects can be relied on. + + A manifest is written (and stays) ``active`` for as long as its backup is + restorable from its own objects. A delete marks it ``deleting`` before it + touches any S3 object, and removes the manifest only once the data plane + has confirmed those objects are gone -- the single-backup half of the same + metadata-before-data ordering :class:`GroupManifestState` keeps for group + backups. A reader that is not the one deleting it -- a disaster-recovery + ``discover``/``import`` on another cluster -- must skip a manifest in this + state rather than treat it as restorable, the same rule + ``import_group_backup_from_bucket`` already applies to a non-``COMMITTED`` + group manifest. A manifest written before deletes marked it reads as + ``active``, which is what it was. + """ + ACTIVE = "active" + DELETING = "deleting" + + class BackupManifest(BaseModel): model_config = ConfigDict(extra="forbid") @@ -304,6 +323,9 @@ class BackupManifest(BaseModel): volume: ManifestVolume dataplane: ManifestDataPlane + #: See BackupManifestState. + state: BackupManifestState = BackupManifestState.ACTIVE + EXPORT_SCHEMA_VERSION = 1 @@ -437,8 +459,9 @@ def delete(config: BackupConfig, backup_id: UUID) -> None: """Remove a manifest. Only for a backup whose data is genuinely gone -- a merge folding it into - its successor. Deleting the manifest of a backup whose objects still exist - turns them into weight nobody can identify or reclaim. + its successor, or a delete that has confirmed the data plane removed its + objects. Deleting the manifest of a backup whose objects still exist turns + them into weight nobody can identify or reclaim. """ try: s3_client(config).delete_object( @@ -447,6 +470,59 @@ def delete(config: BackupConfig, backup_id: UUID) -> None: raise ManifestError(f"Failed to delete manifest for backup {backup_id}") from e +def _read_if_present(config: BackupConfig, backup_id: UUID) -> BackupManifest | None: + """:func:`read`, but ``None`` for a manifest the bucket does not hold.""" + try: + body = s3_client(config).get_object( + Bucket=config.bucket_name, Key=manifest_key(backup_id))["Body"].read() + except ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + return None + raise ManifestError( + f"Failed to read manifest for backup {backup_id} " + f"from {config.bucket_name}") from e + except BotoCoreError as e: + raise ManifestError( + f"Failed to read manifest for backup {backup_id} " + f"from {config.bucket_name}") from e + return _parse(body, manifest_key(backup_id)) + + +def mark_deleting(config: BackupConfig, backup_id: UUID) -> None: + """Mark a backup's manifest DELETING, before any of its S3 objects are + touched -- the single-backup half of the metadata-before-data ordering + :func:`delete_group_backup ` keeps for + group backups. + + A no-op if the manifest already says so (a delete resuming after a + restart or retry), or if it is already gone (a delete that got as far as + removing the manifest before being abandoned -- :func:`delete`, not this, + is the next step on retry, not this one again). + + Raises: + ManifestError: the manifest could not be read or rewritten. + """ + manifest = _read_if_present(config, backup_id) + if manifest is not None and manifest.state != BackupManifestState.DELETING: + write(config, manifest.model_copy(update={"state": BackupManifestState.DELETING})) + + +def mark_active(config: BackupConfig, backup_id: UUID) -> None: + """Revert a backup's manifest to ACTIVE: a delete that did not finish. + + A no-op if the manifest is already active, or already gone (the delete + got as far as removing it before being abandoned -- there is nothing left + to revert, and the FDB record it was abandoned alongside is reverted + separately). + + Raises: + ManifestError: the manifest could not be read or rewritten. + """ + manifest = _read_if_present(config, backup_id) + if manifest is not None and manifest.state != BackupManifestState.ACTIVE: + write(config, manifest.model_copy(update={"state": BackupManifestState.ACTIVE})) + + def _parse(body: bytes, key: str) -> BackupManifest: try: data = json.loads(body) diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index 1889894b43..5b8ac415f0 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -1619,6 +1619,34 @@ def add_backup_restore_task(source_lvol_id, cluster_id, node_id, backup_id, lvol return task.uuid +def add_backup_delete_task(cluster_id, node_id, backup_ids): + """Create the task that deletes one or more backups of the same volume + that nothing chains against, unless an open task already covers them. + + The targets are already DELETING by the time this is called -- + ``backup_controller.delete_backup``/``delete_backups`` claim them via + ``DBController.claim_backup_chain_op`` first -- so, like + ``add_backup_merge_task``, this needs no transactional join of its own: + there is no gap between a check and this task existing left to close. + """ + wanted = set(backup_ids) + for task in db.get_job_tasks_by_function(cluster_id, JobSchedule.FN_BACKUP_DELETE): + if (not task.canceled and task.status != JobSchedule.STATUS_DONE + and wanted <= set((task.function_params or {}).get("backup_ids") or [])): + logger.info(f"Task found, skip adding new task: {task.uuid}") + return False + return _add_task( + JobSchedule.FN_BACKUP_DELETE, + cluster_id, + node_id, + "", + max_retry=constants.BACKUP_TASK_MAX_RETRIES, + function_params={ + "backup_ids": list(backup_ids), + }, + ) + + def add_backup_merge_task(cluster_id, node_id, keep_backup_id, old_backup_id): """Create the task that merges two S3 backups.""" return _add_task( diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index 2c15cc29ef..cf8e023c25 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -1514,14 +1514,25 @@ def _claim_backup_chain_op_tx(self, tr, lvol_id, target_ids, new_status): idx.entry_id(Backup, bytes(key), bytes(value)) for key, value in tr.get_range_startswith(idx.prefix(Backup, (lvol_id,))) ] + # Keyed by the bare uuid, not get_id(): every caller holds a Backup's + # uuid, never its "/" composite id, and a chain + # never spans clusters -- so the uuid alone is already unambiguous + # within one lvol_id's siblings. siblings = {} for sibling_id in sibling_ids: rows = Backup().read_from_db(tr, id=sibling_id) if rows: - siblings[sibling_id] = rows[0] - + siblings[rows[0].uuid] = rows[0] + + target_set = set(target_ids) + # A sibling already MERGING/DELETING blocks the claim UNLESS it is one + # of this call's own targets: re-claiming backups already in the state + # this call is about to set is what makes resuming an interrupted + # delete (the caller is retrying, not racing itself) safe, while a + # DIFFERENT concurrent operation's claim still refuses. blocking = next((b for b in siblings.values() - if b.status in (Backup.STATUS_MERGING, Backup.STATUS_DELETING)), None) + if b.status in (Backup.STATUS_MERGING, Backup.STATUS_DELETING) + and b.uuid not in target_set), None) if blocking is not None: return False, f"backup {blocking.uuid} of this chain is already {blocking.status}" @@ -1553,15 +1564,30 @@ def _claim_backup_chain_op_tx(self, tr, lvol_id, target_ids, new_status): return True, None def claim_backup_chain_op(self, lvol_id, target_ids, new_status): - """Atomically claim one or more backups of ``lvol_id``'s chain for a - destructive operation (merge or delete). - - Refuses, as ``(False, reason)``, if any backup anywhere in the chain -- - not only the targets -- is already ``MERGING``/``DELETING``, or is named - by an active restore's chain (``JobSchedule.restore_chain_s3_id``). On - success, every id in ``target_ids`` is flipped to ``new_status`` and + """Atomically claim one or more backups of ``lvol_id``'s chain (named + by their bare ``uuid``, not ``get_id()``) for a destructive operation + (merge or delete). + + Refuses, as ``(False, reason)``, if any OTHER backup of the chain -- + one not in ``target_ids`` -- is already ``MERGING``/``DELETING``, or if + any backup in the chain (target or not) is named by an active + restore's own chain (``JobSchedule.restore_chain_s3_id``). On success, + every id in ``target_ids`` is flipped to ``new_status`` and ``(True, None)`` is returned. + Idempotent by design, not exclusive between identical claims: a target + already in ``new_status`` -- this call resuming one that claimed it + but did not finish queuing the task that follows -- does not block + itself. Two callers racing to claim the very same ids at once can + therefore BOTH succeed (one commits, the other retries on conflict, + re-reads the now-already-``new_status`` targets, and finds nothing of + its own to object to) -- what this guarantees is that no OTHER, + genuinely different claim can interleave with either, not that only + one of two identical ones wins. Callers rely on idempotency further + downstream (``manifest.mark_deleting`` is a no-op if already marked; + ``add_backup_delete_task`` dedupes against an open task) to make a + repeat claim of the same ids harmless rather than merely admitted. + Assumes ``Backup``'s ``lvol_id`` index is ready, which it already is in any deployment old enough to run this: an index still ``building`` would make this read incomplete rather than merely slow, so this does not fall diff --git a/simplyblock_core/models/job_schedule.py b/simplyblock_core/models/job_schedule.py index 6d6ec7cee7..42c6ff7482 100644 --- a/simplyblock_core/models/job_schedule.py +++ b/simplyblock_core/models/job_schedule.py @@ -80,6 +80,11 @@ class JobSchedule(BaseModel): FN_BACKUP = "s3_backup" FN_BACKUP_RESTORE = "s3_backup_restore" FN_BACKUP_MERGE = "s3_backup_merge" + # Delete one or more backups of the same volume that nothing chains + # against: remove their S3 objects, then their manifests, then their + # records -- metadata-before-data on the way in (claim_backup_chain_op + # marks them DELETING first), data-before-metadata on the way out. + FN_BACKUP_DELETE = "s3_backup_delete" # Consistency-group backup commit: watch the member backups of one group # generation and publish the group manifest once every member completed, # the all-or-nothing barrier. diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index d0f14b0a7e..09c1f03dc3 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -414,6 +414,67 @@ def _run_merge(task): raise TaskDefer("Merge in progress") +def _run_delete(task): + """Delete the backups named by ``function_params["backup_ids"]``: remove + their S3 objects, then their manifests, then their records -- the reverse + of the order ``backup_controller._start_backup_delete`` claimed them in + (record marked DELETING, then manifest marked DELETING, then this task). + + A backup already gone, or no longer DELETING, is skipped: a prior cycle + of this same task already retired it, or (the group-backup path) a + different caller did. + + Single-cycle, unlike backup/restore/merge: ``bdev_lvol_s3_delete`` has no + paired stat RPC to poll in ``rpc_client.py``, so it is treated as + synchronous, exactly as its one existing client method already models it. + """ + backup_ids = task.function_params.get("backup_ids") or [] + if not backup_ids: + raise TaskAbort("Missing backup_ids") + + backups = [] + for backup_id in backup_ids: + try: + backup = db.get_backup_by_id(backup_id) + except KeyError: + continue + if backup.status != Backup.STATUS_DELETING: + continue + backups.append(backup) + + if not backups: + set_result(task, "Nothing left to delete") + return + + snode = _online_node(backups[0].node_id) + rpc_client = snode.rpc_client(timeout=30) + + try: + ret = rpc_client.bdev_lvol_s3_delete([b.s3_id for b in backups]) + except RPCException as e: + _defer_if_busy(e) + raise TaskRetry(f"RPC error: {e}") + if not ret: + raise TaskRetry("bdev_lvol_s3_delete RPC failed") + + # The data is confirmed gone: metadata follows data on the way out, the + # reverse of how the claim marked intent before any of this ran. + for backup in backups: + try: + backup_controller.delete_manifest(backup) + except (ManifestError, PreconditionError) as e: + # The S3 delete already happened and is not reversible, so the + # task cannot be aborted here -- retry the manifest/record cleanup. + raise TaskRetry(f"Delete done, manifest cleanup failed: {e}") + + backup_controller.cleanup_backup_kms_keys([backup]) + backup_events.backup_deleted(backup.cluster_id, backup.node_id, backup) + backup.remove(db.kv_store) + + set_result(task, "Backups deleted") + logger.info(f"Deleted backups: {', '.join(b.uuid for b in backups)}") + + def _run_group_backup_commit(task): """Commit a consistency-group backup once its members are done. @@ -468,6 +529,7 @@ def _run_group_backup_delete(task): JobSchedule.FN_BACKUP: _run_backup, JobSchedule.FN_BACKUP_RESTORE: _run_restore, JobSchedule.FN_BACKUP_MERGE: _run_merge, + JobSchedule.FN_BACKUP_DELETE: _run_delete, JobSchedule.FN_GROUP_BACKUP_COMMIT: _run_group_backup_commit, JobSchedule.FN_GROUP_BACKUP_DELETE: _run_group_backup_delete, } @@ -558,6 +620,31 @@ def finalize_resource(task): gb.error_message = reason gb.write_to_db() + elif task.function_name == JobSchedule.FN_BACKUP_DELETE: + # The delete did not finish (timeout, retry ceiling, abort, + # cancellation): every backup this task still finds DELETING had its + # S3 objects left untouched -- the RPC either never ran or never + # confirmed success, and _run_delete only removes a manifest/record + # after it has -- so each is reverted to COMPLETED, record and + # manifest both, the same way an unfinished merge leaves its old + # backup intact. + for backup_id in task.function_params.get("backup_ids") or []: + try: + backup = db.get_backup_by_id(backup_id) + except KeyError: + continue + if backup.status != Backup.STATUS_DELETING: + continue + try: + backup_controller.revert_manifest_deleting(backup) + except (ManifestError, PreconditionError, KeyError) as e: + logger.error( + f"Backup {backup.uuid} reverted to COMPLETED, but its " + f"manifest could not be marked active and still says " + f"deleting: {e}") + backup.status = Backup.STATUS_COMPLETED + backup.write_to_db() + elif task.function_name == JobSchedule.FN_GROUP_BACKUP_DELETE: group_backup_id = task.function_params.get("group_backup_id") if not group_backup_id: @@ -582,7 +669,8 @@ def finalize_resource(task): #: "max retry reached (8/8)" and would end any other task older than the #: backup timeout (2026-09-29). BACKUP_FUNCTIONS = (JobSchedule.FN_BACKUP, JobSchedule.FN_BACKUP_RESTORE, - JobSchedule.FN_BACKUP_MERGE, JobSchedule.FN_GROUP_BACKUP_COMMIT, + JobSchedule.FN_BACKUP_MERGE, JobSchedule.FN_BACKUP_DELETE, + JobSchedule.FN_GROUP_BACKUP_COMMIT, JobSchedule.FN_GROUP_BACKUP_DELETE) diff --git a/tests/integration/backup/test_delete_backup.py b/tests/integration/backup/test_delete_backup.py index c0cc3b7a5d..661fc4bce2 100644 --- a/tests/integration/backup/test_delete_backup.py +++ b/tests/integration/backup/test_delete_backup.py @@ -1,26 +1,41 @@ -"""delete_backup removes one backup record, and refuses what would break a restore, -against real FoundationDB. +"""Deleting (or merging away) one backup, and deleting every backup of a +volume, against real FoundationDB. -Deleting a backup another one chains against makes every descendant -unrestorable, and deleting a member of a group backup that still exists tears -that group backup. Both are refused with an exception rather than reported in a -return value, so a caller cannot forget to look. +A backup with nothing chaining against it is claimed for deletion: its +status moves to DELETING, its manifest is marked DELETING in the bucket, and +a task is queued to remove its S3 objects. A backup something chains against +is instead merged into that successor -- deleting it and merging it are the +same request once a successor exists (``trigger_merge``). Both forms go +through the same chain-wide guard (``DBController.claim_backup_chain_op``), +so a chain already claimed by another merge, delete, or an active restore +refuses a second one. -The database is real. The only boundary patched is the storage node's RPC -client, which the integration tier never reaches a real node for. +The database is real. The bucket is the one boundary faked: the S3 client the +manifest functions use is an in-memory one, so manifest marking runs for +real over it. """ -import uuid as uuid_mod +import io import zlib -from unittest.mock import patch +from uuid import UUID import pytest +from botocore.exceptions import ClientError from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import ( + BackupManifest, + BackupManifestState, + ManifestDataPlane, + ManifestSource, + ManifestVolume, +) from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup, GroupBackup -from simplyblock_core.models.storage_node import StorageNode -from simplyblock_core.rpc_client import RPCException +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.job_schedule import JobSchedule def _uuid(name: str) -> str: @@ -29,48 +44,89 @@ def _uuid(name: str) -> str: CLUSTER = _uuid("cluster") LVOL = _uuid("lvol") +NODE = _uuid("node") +BUCKET = "simplyblock-backup-cluster" + + +def _config(): + return BackupConfig.model_validate({"bucket_name": BUCKET, "region": "eu-central-1"}) @pytest.fixture def db(): - return DBController() + db = DBController() + c = Cluster() + c.uuid = CLUSTER + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return db + + +class _Bucket: + """An in-memory S3 bucket, behind the three object calls the manifest + functions make.""" + + def __init__(self): + self.objects: dict = {} + + def put_object(self, Bucket, Key, Body, **_): + self.objects[Key] = Body + + def get_object(self, Bucket, Key): + if Key not in self.objects: + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") + return {"Body": io.BytesIO(self.objects[Key])} + + def delete_object(self, Bucket, Key): + self.objects.pop(Key, None) @pytest.fixture -def node(db): - """An online node whose RPC client is patched: the node has no reachable - address, and the S3 delete is the only call made to it.""" - n = StorageNode() - n.uuid = str(uuid_mod.uuid4()) - n.cluster_id = CLUSTER - n.status = StorageNode.STATUS_ONLINE - n.hostname = "delete-backup-host" - n.mgmt_ip = "127.0.0.1" - n.write_to_db(db.kv_store) - with patch.object(StorageNode, "rpc_client") as rpc: - n.rpc = rpc - yield n - - -def _backup(db, node, name, prev="", group_backup_id=""): +def bucket(monkeypatch): + """The backup manifests in the bucket. The manifest functions run for + real over it; only the S3 client is faked.""" + b = _Bucket() + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + return b + + +def _manifest(backup_id: str) -> BackupManifest: + return BackupManifest( + backup_id=UUID(backup_id), + s3_id=zlib.crc32(backup_id.encode()) % 1000, + created_at=100, + completed_at=200, + size=4096, + source=ManifestSource(cluster_id=UUID(CLUSTER), node_id=UUID(NODE)), + volume=ManifestVolume( + lvol_id=UUID(LVOL), lvol_name="vol", snapshot_id=UUID(_uuid("snap")), + snapshot_name="snap", size=4096), + dataplane=ManifestDataPlane(), + ) + + +def _backup(db, bucket, name, prev="", group_backup_id="", status=Backup.STATUS_COMPLETED): b = Backup() b.uuid = _uuid(name) b.cluster_id = CLUSTER b.lvol_id = LVOL - b.node_id = node.uuid + b.node_id = NODE b.s3_id = zlib.crc32(name.encode()) % 1000 b.prev_backup_id = _uuid(prev) if prev else "" b.group_backup_id = group_backup_id - b.status = Backup.STATUS_COMPLETED + b.status = status + b.location = _config().location().model_dump(mode="json") b.write_to_db(db.kv_store) + if status == Backup.STATUS_COMPLETED: + backup_manifest.write(_config(), _manifest(b.uuid)) return b -def _group_backup(db): +def _group_backup(db, status=GroupBackup.STATUS_COMPLETED): gb = GroupBackup() gb.uuid = _uuid("gb") gb.cluster_id = CLUSTER - gb.status = GroupBackup.STATUS_COMPLETED + gb.status = status gb.write_to_db(db.kv_store) return gb @@ -83,32 +139,103 @@ def _exists(db, backup): return False -def test_a_backup_nothing_depends_on_is_deleted(db, node): - leaf = _backup(db, node, "leaf") +def _status(db, backup): + return db.get_backup_by_id(backup.uuid).status + + +def _manifest_state(bucket, backup): + return backup_manifest._parse( + bucket.objects[backup_manifest.manifest_key(UUID(backup.uuid))], + "test").state + + +def _delete_tasks(db, backup_id=None): + tasks = [t for t in db.get_job_tasks_by_function(CLUSTER, JobSchedule.FN_BACKUP_DELETE) + if t.status != JobSchedule.STATUS_DONE and not t.canceled] + if backup_id is None: + return tasks + return [t for t in tasks if backup_id in (t.function_params or {}).get("backup_ids", [])] + + +def _merge_tasks(db, old_backup_id=None): + tasks = [t for t in db.get_job_tasks_by_function(CLUSTER, JobSchedule.FN_BACKUP_MERGE) + if t.status != JobSchedule.STATUS_DONE and not t.canceled] + if old_backup_id is None: + return tasks + return [t for t in tasks if (t.function_params or {}).get("old_backup_id") == old_backup_id] + + +def _restore_task(db, chain_s3_ids): + """A live FN_BACKUP_RESTORE task reading ``chain_s3_ids``, so the restore + index picks it up.""" + task, err = db.create_backup_restore_task_if_chain_clear( + LVOL, CLUSTER, NODE, + function_params={"backup_id": "", "lvol_name": "x", "lvol_id": "", + "chain_ids": chain_s3_ids, "s3_config": None}, + max_retry=1) + assert err is None, err + return task + + +def test_a_leaf_backup_is_claimed_and_queued_for_deletion(db, bucket): + leaf = _backup(db, bucket, "leaf") + assert backup_controller.delete_backup(leaf.uuid) is None - assert not _exists(db, leaf) + assert _exists(db, leaf) + assert _status(db, leaf) == Backup.STATUS_DELETING + assert _manifest_state(bucket, leaf) == BackupManifestState.DELETING + assert _delete_tasks(db, leaf.uuid) + + +def test_resuming_an_already_deleting_backup_requeues_without_reclaiming(db, bucket): + leaf = _backup(db, bucket, "leaf") + backup_controller.delete_backup(leaf.uuid) + first_tasks = {t.uuid for t in _delete_tasks(db, leaf.uuid)} + + backup_controller.delete_backup(leaf.uuid) + + assert _status(db, leaf) == Backup.STATUS_DELETING + assert {t.uuid for t in _delete_tasks(db, leaf.uuid)} == first_tasks + + +def test_a_backup_with_a_completed_successor_is_merged(db, bucket): + base = _backup(db, bucket, "base") + nxt = _backup(db, bucket, "next", prev="base") + + assert backup_controller.delete_backup(base.uuid) is None + + assert _status(db, base) == Backup.STATUS_MERGING + assert _status(db, nxt) == Backup.STATUS_COMPLETED + assert _merge_tasks(db, base.uuid) + + +def test_a_backup_with_an_incomplete_successor_is_refused(db, bucket): + base = _backup(db, bucket, "base") + _backup(db, bucket, "next", prev="base", status=Backup.STATUS_IN_PROGRESS) -def test_a_backup_another_chains_against_is_refused(db, node): - base = _backup(db, node, "base") - _backup(db, node, "next", prev="base") - with pytest.raises(PreconditionError, match="predecessor"): + with pytest.raises(PreconditionError, match="completed"): backup_controller.delete_backup(base.uuid) - assert _exists(db, base) + assert _status(db, base) == Backup.STATUS_COMPLETED -def test_a_member_of_a_group_backup_that_exists_is_refused(db, node): + +def test_a_member_of_a_group_backup_that_exists_is_refused(db, bucket): gb = _group_backup(db) - member = _backup(db, node, "member", group_backup_id=gb.get_id()) + member = _backup(db, bucket, "member", group_backup_id=gb.get_id()) + with pytest.raises(PreconditionError, match="group backup"): backup_controller.delete_backup(member.uuid) - assert _exists(db, member) + + assert _status(db, member) == Backup.STATUS_COMPLETED -def test_a_member_whose_group_backup_is_gone_is_deleted(db, node): - orphan = _backup(db, node, "orphan", group_backup_id=f"{CLUSTER}/{_uuid('gone')}") +def test_a_member_whose_group_backup_is_gone_is_queued_for_deletion(db, bucket): + orphan = _backup(db, bucket, "orphan", group_backup_id=f"{CLUSTER}/{_uuid('gone')}") + backup_controller.delete_backup(orphan.uuid) - assert not _exists(db, orphan) + + assert _status(db, orphan) == Backup.STATUS_DELETING def test_an_unknown_backup_raises(db): @@ -116,9 +243,97 @@ def test_an_unknown_backup_raises(db): backup_controller.delete_backup(_uuid("nope")) -def test_a_failed_s3_delete_still_deletes_the_record(db, node): - """The S3 delete is best effort: the data plane does not implement it yet.""" - leaf = _backup(db, node, "leaf") - node.rpc.return_value.bdev_lvol_s3_delete.side_effect = RPCException("no such method") +def test_a_chain_already_being_merged_refuses_a_delete_of_another_backup(db, bucket): + """Two leaves of the same volume's chain, one already MERGING -- claiming + the other must see it and refuse.""" + base = _backup(db, bucket, "base") + nxt = _backup(db, bucket, "next", prev="base") + other_leaf = _backup(db, bucket, "other") + ok, reason = backup_controller.trigger_merge(nxt, base) + assert ok, reason + + with pytest.raises(PreconditionError, match="already"): + backup_controller.delete_backup(other_leaf.uuid) + + assert _status(db, other_leaf) == Backup.STATUS_COMPLETED + + +def test_a_chain_read_by_an_active_restore_refuses_a_delete(db, bucket): + leaf = _backup(db, bucket, "leaf") + _restore_task(db, [leaf.s3_id]) + + with pytest.raises(PreconditionError, match="restore"): + backup_controller.delete_backup(leaf.uuid) + + assert _status(db, leaf) == Backup.STATUS_COMPLETED + + +def test_a_chain_read_by_an_active_restore_refuses_a_new_restore_task(db, bucket): + """create_backup_restore_task_if_chain_clear itself refuses once a sibling + is DELETING -- the other half of the same guard.""" + leaf = _backup(db, bucket, "leaf") backup_controller.delete_backup(leaf.uuid) - assert not _exists(db, leaf) + + task, err = db.create_backup_restore_task_if_chain_clear( + LVOL, CLUSTER, NODE, + function_params={"backup_id": leaf.uuid, "lvol_name": "x", "lvol_id": "", + "chain_ids": [leaf.s3_id], "s3_config": None}, + max_retry=1) + + assert task is None + assert "deleting" in err + + +def test_delete_backups_refuses_while_a_backup_is_still_being_created(db, bucket): + _backup(db, bucket, "done") + _backup(db, bucket, "uploading", status=Backup.STATUS_IN_PROGRESS) + + ok, err = backup_controller.delete_backups(LVOL) + + assert not ok + assert "still being created" in err + + +def test_delete_backups_refuses_while_a_backup_is_merging(db, bucket): + base = _backup(db, bucket, "base") + nxt = _backup(db, bucket, "next", prev="base") + ok, reason = backup_controller.trigger_merge(nxt, base) + assert ok, reason + + ok, err = backup_controller.delete_backups(LVOL) + + assert not ok + assert "merged" in err + + +def test_delete_backups_refuses_for_a_live_group_member(db, bucket): + gb = _group_backup(db) + _backup(db, bucket, "member", group_backup_id=gb.get_id()) + + ok, err = backup_controller.delete_backups(LVOL) + + assert not ok + assert "group backup" in err + + +def test_delete_backups_claims_completed_backups_and_removes_failed_ones_directly(db, bucket): + completed = _backup(db, bucket, "completed") + failed = _backup(db, bucket, "failed", status=Backup.STATUS_FAILED) + + ok, err = backup_controller.delete_backups(LVOL) + + assert ok, err + assert _status(db, completed) == Backup.STATUS_DELETING + assert _manifest_state(bucket, completed) == BackupManifestState.DELETING + assert _delete_tasks(db, completed.uuid) + assert not _exists(db, failed) + + +def test_delete_backups_is_a_noop_resume_once_everything_is_already_deleting(db, bucket): + _backup(db, bucket, "leaf") + ok, err = backup_controller.delete_backups(LVOL) + assert ok, err + + ok, err = backup_controller.delete_backups(LVOL) + + assert ok, err diff --git a/tests/unit/backup/test_manifest_delete_state.py b/tests/unit/backup/test_manifest_delete_state.py new file mode 100644 index 0000000000..7a3cdc762f --- /dev/null +++ b/tests/unit/backup/test_manifest_delete_state.py @@ -0,0 +1,139 @@ +"""Marking a single-backup manifest DELETING/ACTIVE, ahead of actually +removing anything. + +A delete marks a backup's manifest DELETING before any of its S3 objects are +touched, and reverts it to ACTIVE if the delete does not finish -- the +single-backup half of the metadata-before-data ordering the group manifest's +``GroupManifestState`` already keeps. Both functions are read-modify-write +over the bucket alone; no database is involved. + +The S3 client is the one boundary faked, as an in-memory bucket, following +the same pattern as ``test_group_manifest_delete.py``. +""" +import io +import zlib +from uuid import UUID + +import pytest +from botocore.exceptions import ClientError + +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import ( + BackupManifest, + BackupManifestState, + ManifestDataPlane, + ManifestError, + ManifestSource, + ManifestVolume, +) +from simplyblock_core.models.backup_config import BackupConfig + + +def _id(name: str) -> UUID: + return UUID(f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000") + + +BACKUP = _id("b-1") +CONFIG = BackupConfig.model_validate({"bucket_name": "backups", "region": "eu-central-1"}) +KEY = backup_manifest.manifest_key(BACKUP) + + +class _Bucket: + """The three object calls mark_deleting/mark_active make.""" + + def __init__(self, fail=False): + self.objects: dict = {} + self.fail = fail + + def _check(self, op): + if self.fail: + raise ClientError({"Error": {"Code": "InternalError"}}, op) + + def put_object(self, Bucket, Key, Body, **_): + self._check("PutObject") + self.objects[Key] = Body + + def get_object(self, Bucket, Key): + self._check("GetObject") + if Key not in self.objects: + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") + return {"Body": io.BytesIO(self.objects[Key])} + + def delete_object(self, Bucket, Key): + self._check("DeleteObject") + self.objects.pop(Key, None) + + +@pytest.fixture +def bucket(monkeypatch): + b = _Bucket() + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + return b + + +def _store(state=BackupManifestState.ACTIVE): + backup_manifest.write(CONFIG, BackupManifest( + backup_id=BACKUP, + s3_id=1, + created_at=100, + completed_at=200, + size=4096, + source=ManifestSource(cluster_id=_id("cluster"), node_id=_id("node")), + volume=ManifestVolume( + lvol_id=_id("lvol"), lvol_name="vol", snapshot_id=_id("snap"), + snapshot_name="snap", size=4096), + dataplane=ManifestDataPlane(), + state=state, + )) + + +def test_mark_deleting_rewrites_an_active_manifest(bucket): + _store(BackupManifestState.ACTIVE) + + backup_manifest.mark_deleting(CONFIG, BACKUP) + + assert backup_manifest.read(CONFIG, BACKUP).state == BackupManifestState.DELETING + + +def test_mark_deleting_is_a_noop_if_already_deleting(bucket): + _store(BackupManifestState.DELETING) + + backup_manifest.mark_deleting(CONFIG, BACKUP) + + assert backup_manifest.read(CONFIG, BACKUP).state == BackupManifestState.DELETING + + +def test_mark_deleting_is_a_noop_if_the_manifest_is_already_gone(bucket): + backup_manifest.mark_deleting(CONFIG, BACKUP) + + assert bucket.objects == {} + + +def test_mark_active_reverts_a_deleting_manifest(bucket): + _store(BackupManifestState.DELETING) + + backup_manifest.mark_active(CONFIG, BACKUP) + + assert backup_manifest.read(CONFIG, BACKUP).state == BackupManifestState.ACTIVE + + +def test_mark_active_is_a_noop_if_already_active(bucket): + _store(BackupManifestState.ACTIVE) + + backup_manifest.mark_active(CONFIG, BACKUP) + + assert backup_manifest.read(CONFIG, BACKUP).state == BackupManifestState.ACTIVE + + +def test_mark_active_is_a_noop_if_the_manifest_is_already_gone(bucket): + backup_manifest.mark_active(CONFIG, BACKUP) + + assert bucket.objects == {} + + +def test_a_bucket_that_cannot_be_read_is_a_manifest_error(monkeypatch): + b = _Bucket(fail=True) + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + + with pytest.raises(ManifestError): + backup_manifest.mark_deleting(CONFIG, BACKUP) From 2acbccbde0f69fb8455eb3825cd64e1e8cff2cd4 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 13 Nov 2026 01:49:55 +0100 Subject: [PATCH 03/13] Expose single-backup delete via the v2 API and sbctl CLI DELETE /clusters/{cluster_id}/backups/{backup_id} (route name clusters:backups:delete-one) calls backup_controller.delete_backup, mapping PreconditionError to 409 -- the same convention delete_group_backup already uses. sbctl backup delete-backup is the CLI equivalent (backup__delete_backup in clibase.py, mirroring group_backup__delete). Both per-volume bulk-delete endpoints (the current one under storage-pools/.../volumes/{id}/backups, and the deprecated /clusters/{id}/backups/{volume_id}) now return 202 instead of 204: deletion is asynchronous since delete_backups only claims and queues a task, it does not finish before returning. Their error status moves from 400 to 409 for the same reason delete_group_backup uses 409 over HTTPException's usual 400 -- these are precondition conflicts, not malformed requests. Running the full unit suite after this surfaced two real gaps the new task type exposed: TaskDTO.function_name's Literal type didn't list "s3_backup_delete" (every /tasks/ list request with one in it 500'd), and test_backup_family_is_owned's exact-set assertion over tasks_runner_backup.SPEC.function_names didn't expect it either. Both fixed. Full unit suite (6965 tests) and lint/types green. --- simplyblock_cli/cli-reference.yaml | 7 ++++++ simplyblock_cli/cli.py | 7 ++++++ simplyblock_cli/clibase.py | 12 ++++++++-- simplyblock_web/api/v2/_dtos.py | 1 + simplyblock_web/api/v2/cluster/backup.py | 24 +++++++++++++++---- .../cluster/storage_pool/volume/__init__.py | 12 ++++++---- tests/unit/test_replication_incident_0929.py | 4 ++-- 7 files changed, 55 insertions(+), 12 deletions(-) diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 2b907415aa..4019929ad2 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -3364,6 +3364,13 @@ commands: help: "The logical volume id." dest: lvol_id type: str + - name: delete-backup + help: "Delete one backup. A backup nothing chains against is deleted; one something chains against is merged into that successor instead." + arguments: + - name: "backup_id" + help: "The backup id." + dest: backup_id + type: str - name: restore help: "Restore a backup to a new logical volume." arguments: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 0d21c5fcb1..d29262f012 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -1310,6 +1310,7 @@ def init_backup(self): subparser = self.add_command('backup', 'Backup Commands') self.init_backup__list(subparser) self.init_backup__delete(subparser) + self.init_backup__delete_backup(subparser) self.init_backup__restore(subparser) self.init_backup__export(subparser) self.init_backup__discover(subparser) @@ -1330,6 +1331,10 @@ def init_backup__delete(self, subparser): subcommand = self.add_sub_command(subparser, 'delete', 'Delete all backups for a logical volume.') subcommand.add_argument('lvol_id', help='The logical volume id.', type=str) + def init_backup__delete_backup(self, subparser): + subcommand = self.add_sub_command(subparser, 'delete-backup', 'Delete one backup. A backup nothing chains against is deleted; one something chains against is merged into that successor instead.') + subcommand.add_argument('backup_id', help='The backup id.', type=str) + def init_backup__restore(self, subparser): subcommand = self.add_sub_command(subparser, 'restore', 'Restore a backup to a new logical volume.') subcommand.add_argument('backup_id', help='The volume backup id.', type=str) @@ -2017,6 +2022,8 @@ def run(self): ret = self.backup__list(sub_command, args) elif sub_command in ['delete']: ret = self.backup__delete(sub_command, args) + elif sub_command in ['delete-backup']: + ret = self.backup__delete_backup(sub_command, args) elif sub_command in ['restore']: ret = self.backup__restore(sub_command, args) elif sub_command in ['export']: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 3a9b470dc6..0bd0fbb439 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -1773,8 +1773,16 @@ def backup__delete(self, sub_command, args): if error: print(f"Error: {error}") return False - print("Backups deleted") - return True + return f"Backups of volume {args.lvol_id} are being deleted" + + def backup__delete_backup(self, sub_command, args): + try: + backup_controller.delete_backup(args.backup_id) + except KeyError: + return f"Delete failed: backup {args.backup_id} not found" + except PreconditionError as e: + return f"Delete failed: {e}" + return f"Backup {args.backup_id} is being deleted" def backup__restore(self, sub_command, args): try: diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index c946c655bf..b33f5334ed 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -150,6 +150,7 @@ "s3_backup", "s3_backup_restore", "s3_backup_merge", + "s3_backup_delete", "group_backup_commit", "group_backup_delete", "cluster_expand", diff --git a/simplyblock_web/api/v2/cluster/backup.py b/simplyblock_web/api/v2/cluster/backup.py index aa5e123f9d..068c2a3e94 100644 --- a/simplyblock_web/api/v2/cluster/backup.py +++ b/simplyblock_web/api/v2/cluster/backup.py @@ -9,6 +9,7 @@ from simplyblock_core.controllers.backup import policy as backup_policy from simplyblock_core.controllers.backup.manifest import ManifestError from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup_config import S3Credentials from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.lvol_model import LVol @@ -173,8 +174,8 @@ def _lookup_lvol_in_cluster(volume_id: str, cluster: ClusterModel) -> LVol: @api.delete( '/{volume_id}', name='clusters:backups:delete', - status_code=204, - responses={204: {"content": None}}, + status_code=202, + responses={202: {"content": None}}, deprecated=True, summary='Deprecated — delete all backups for a volume', description=( @@ -187,8 +188,8 @@ def delete_backups(cluster: Cluster, volume_id: UUID) -> Response: volume = _lookup_lvol_in_cluster(str(volume_id), cluster) success, error = backup_controller.delete_backups(volume.get_id()) if error: - raise HTTPException(400, error) - return Response(status_code=204) + raise HTTPException(409, error) + return Response(status_code=202) # Backup policies @@ -286,4 +287,19 @@ def get_backup(cluster: Cluster, backup: BackupResource) -> BackupDTO: return BackupDTO.from_model(backup) +@instance_api.delete('/', name='clusters:backups:delete-one', + status_code=202, responses={202: {"content": None}}) +def delete_backup(cluster: Cluster, backup: BackupResource) -> Response: + """Delete one backup. A backup nothing chains against is claimed for + deletion: its record and manifest are marked deleting, and a task + removes its S3 objects and then both. A backup something chains against + is instead merged into that successor -- deleting it and merging it are + the same request once a successor exists.""" + try: + backup_controller.delete_backup(backup.uuid) + except PreconditionError as e: + raise HTTPException(409, str(e)) from e + return Response(status_code=202) + + api.include_router(instance_api) diff --git a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py index b5f1f282ae..11b9a7d4cd 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py @@ -446,14 +446,18 @@ def create_backup( @instance_api.delete( '/backups', name='clusters:storage-pools:volumes:backups:delete', - status_code=204, - responses={204: {"content": None}}, + status_code=202, + responses={202: {"content": None}}, ) def delete_backups(cluster: Cluster, pool: StoragePool, volume: Volume) -> Response: + """Start deleting every backup of this volume. Completed backups are + claimed and a task removes their S3 objects and then their records and + manifests; the request returns once they are claimed, not once they are + gone.""" success, error = backup_controller.delete_backups(volume.get_id()) if error: - raise HTTPException(400, error) - return Response(status_code=204) + raise HTTPException(409, error) + return Response(status_code=202) api.include_router(replication_collection_api) instance_api.include_router(replication_api, prefix='/replication') diff --git a/tests/unit/test_replication_incident_0929.py b/tests/unit/test_replication_incident_0929.py index 0c54507030..b038a6ad68 100644 --- a/tests/unit/test_replication_incident_0929.py +++ b/tests/unit/test_replication_incident_0929.py @@ -286,8 +286,8 @@ def test_backup_family_is_owned(self): self.assertEqual( set(tasks_runner_backup.SPEC.function_names), {JobSchedule.FN_BACKUP, JobSchedule.FN_BACKUP_RESTORE, - JobSchedule.FN_BACKUP_MERGE, JobSchedule.FN_GROUP_BACKUP_COMMIT, - JobSchedule.FN_GROUP_BACKUP_DELETE}, + JobSchedule.FN_BACKUP_MERGE, JobSchedule.FN_BACKUP_DELETE, + JobSchedule.FN_GROUP_BACKUP_COMMIT, JobSchedule.FN_GROUP_BACKUP_DELETE}, ) From 93cc2b7706f9166fdd9ad31d3990b8caacc295d6 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 13 Nov 2026 02:01:06 +0100 Subject: [PATCH 04/13] Add sbctl cluster check-backups, the backup/manifest consistency checker The backup analog of check_indices: controller.check_backups walks every COMPLETED/DELETING backup in a cluster (or every cluster) and its manifest in both directions. A record whose manifest is missing or disagrees on state/s3_id/prev_backup_id is reported, and --repair rewrites it from the record (today's source of truth). A manifest naming no known record is reported but never auto-repaired -- it may be an unimported disaster-recovery backup, which looks identical to drift from here. MERGED/MERGING backups are explicitly excluded: the merge path doesn't mark manifest state at all, so a leftover manifest there is a known gap in that path, not something this check can resolve. manifest._read_if_present is now public (read_if_present) since check_backups needs the same "read or None" shape mark_deleting/mark_active already used internally. sbctl cluster check-backups [--cluster-id] [--repair] wires it into the CLI, mirroring check-indices. New integration test file (real FDB, fake in-memory bucket) covering the clean case, missing/disagreeing/orphaned manifests, the MERGED exclusion, and cluster scoping -- reviewed but, like the rest of this branch's integration coverage, not run against real FDB in this sandbox. --- simplyblock_cli/cli-reference.yaml | 19 ++ simplyblock_cli/cli.py | 8 + simplyblock_cli/clibase.py | 4 + simplyblock_core/cluster_ops.py | 11 + .../controllers/backup/controller.py | 138 +++++++++++++ .../controllers/backup/manifest.py | 6 +- .../integration/backup/test_check_backups.py | 189 ++++++++++++++++++ 7 files changed, 372 insertions(+), 3 deletions(-) create mode 100644 tests/integration/backup/test_check_backups.py diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 4019929ad2..c5c25ee353 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1763,6 +1763,25 @@ commands: dest: repair type: bool action: store_true + - name: check-backups + help: "Verifies every backup's FDB record against its S3 manifest." + usage: > + Checks, in both directions, that every COMPLETED or DELETING backup has a manifest whose state + and s3_id/prev_backup_id agree with the record, and that every ACTIVE manifest in the bucket + names a record that still exists. Read-only unless `--repair` is given, which rewrites a + missing or disagreeing manifest from the record (today's source of truth). An orphaned + manifest - one naming no known record - is never auto-repaired: it may be a disaster-recovery + backup nothing has imported yet. Deployment-wide unless --cluster-id is given. + arguments: + - name: "--cluster-id" + help: "Limit the check to one cluster. Omit to check every cluster." + dest: cluster_id + type: str + - name: "--repair" + help: "Rewrite manifests that are missing or disagree with their record, instead of only reporting them." + dest: repair + type: bool + action: store_true - name: graceful-shutdown help: "Initiates a graceful shutdown of a cluster's storage nodes." arguments: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index d29262f012..1f1cb48125 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -399,6 +399,7 @@ def init_cluster(self): self.init_cluster__build_indices(subparser) self.init_cluster__index_state(subparser) self.init_cluster__check_indices(subparser) + self.init_cluster__check_backups(subparser) self.init_cluster__graceful_shutdown(subparser) self.init_cluster__restart(subparser) self.init_cluster__graceful_startup(subparser) @@ -645,6 +646,11 @@ def init_cluster__check_indices(self, subparser): subcommand = self.add_sub_command(subparser, 'check-indices', 'Verifies the database\'s secondary indices.') subcommand.add_argument('--repair', help='Write back the entries that can be derived from the records, and clear orphaned ones, instead of only reporting them.', dest='repair', action='store_true') + def init_cluster__check_backups(self, subparser): + subcommand = self.add_sub_command(subparser, 'check-backups', 'Verifies every backup\'s FDB record against its S3 manifest.') + subcommand.add_argument('--cluster-id', help='Limit the check to one cluster. Omit to check every cluster.', type=str, dest='cluster_id') + subcommand.add_argument('--repair', help='Rewrite manifests that are missing or disagree with their record, instead of only reporting them.', dest='repair', action='store_true') + def init_cluster__graceful_shutdown(self, subparser): subcommand = self.add_sub_command(subparser, 'graceful-shutdown', 'Initiates a graceful shutdown of a cluster\'s storage nodes.') subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list @@ -1765,6 +1771,8 @@ def run(self): ret = self.cluster__index_state(sub_command, args) elif sub_command in ['check-indices']: ret = self.cluster__check_indices(sub_command, args) + elif sub_command in ['check-backups']: + ret = self.cluster__check_backups(sub_command, args) elif sub_command in ['graceful-shutdown']: ret = self.cluster__graceful_shutdown(sub_command, args) elif sub_command in ['restart']: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 0bd0fbb439..04c41790f1 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -767,6 +767,10 @@ def cluster__index_state(self, sub_command, args): def cluster__check_indices(self, sub_command, args): return cluster_ops.check_indices(repair=args.repair) + def cluster__check_backups(self, sub_command, args): + return cluster_ops.check_backups( + cluster_id=getattr(args, 'cluster_id', None), repair=args.repair) + def cluster__graceful_shutdown(self, sub_command, args): cluster_ops.cluster_grace_shutdown(args.cluster_id) return True diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index f154629c80..f2a4ce0b60 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -52,6 +52,7 @@ tasks_controller, tcp_ports_events, ) +from simplyblock_core.controllers.backup import controller as backup_controller from simplyblock_core.controllers.backup import device as backup_device from simplyblock_core.controllers.cluster_expansion.planner import ( RemoteTripletPlacementError, @@ -4323,6 +4324,16 @@ def check_indices(repair=False) -> bool: return findings['unresolved'] == 0 +def check_backups(cluster_id=None, repair=False) -> bool: + """Verify every backup's FDB record agrees with its S3 manifest, in both + directions, and report the drift. The backup analog of + :func:`check_indices`. + + Deployment-wide when ``cluster_id`` is omitted. + """ + return backup_controller.check_backups(cluster_id=cluster_id, repair=repair) + + def cluster_grace_startup(cl_id, clear_data=False, spdk_image=None) -> None: get_cluster = db_controller.get_cluster_by_id(cl_id) # ensure exists diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py index 19506ec8b5..7e69998621 100644 --- a/simplyblock_core/controllers/backup/controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -1391,6 +1391,144 @@ def import_from_bucket(config: BackupConfig, cluster_id=None) -> int: cluster_id=cluster_id) +#: Backup statuses whose manifest this function checks, and whose manifest +#: --repair can re-derive from the record. MERGED and MERGING are excluded: +#: the merge path does not mark a manifest state at all (it writes the +#: survivor's and deletes the old one only once the merge itself is done), +#: so neither a MERGED backup's manifest existing nor its absence is a +#: reliable signal here -- that is a gap in the merge path itself, not one +#: this check can resolve by rewriting anything. +_MANIFEST_CHECKED_STATUSES = (Backup.STATUS_COMPLETED, Backup.STATUS_DELETING) + + +def check_backups(cluster_id=None, repair=False) -> bool: + """Verify every backup's FDB record agrees with its S3 manifest, in both + directions, and report the drift. The backup analog of + ``cluster_ops.check_indices``. + + For ``cluster_id`` (or every cluster with a usable backup configuration, + deployment-wide): + + * every COMPLETED or DELETING backup must have a manifest, whose + ``state`` matches (ACTIVE for COMPLETED, DELETING for DELETING) and + whose ``s3_id``/``prev_backup_id`` agree with the record; + * every ACTIVE manifest the bucket holds must name a backup record that + still exists in some status -- one that does not is an orphan, + reported but never repaired: it may be a disaster-recovery backup + nothing has imported yet, which looks identical to genuine drift from + here, and there is no record to derive a fix from either way. + + ``--repair`` rewrites a missing or disagreeing manifest from the record, + which is today's single source of truth: the project's own roadmap is to + make the bucket canonical instead (see ``controller.build_manifest``), + but that is not the status quo this function can assume. + + Returns ``True`` if nothing is left unresolved. + """ + clusters = ( + [db_controller.get_cluster_by_id(cluster_id)] + if cluster_id else db_controller.get_clusters()) + + problems = 0 + repaired = 0 + orphans = 0 + + for cluster in clusters: + try: + config = cluster.get_backup_config() + except ValueError: + continue # nothing of its own to check + + all_backups = db_controller.get_backups(cluster.get_id()) + known_s3_ids = set() + for backup in all_backups: + try: + if backup.get_location() == config.location(): + known_s3_ids.add(backup.s3_id) + except ValueError: + continue + + for backup in all_backups: + if backup.status not in _MANIFEST_CHECKED_STATUSES: + continue + try: + if backup.get_location() != config.location(): + continue # lives in a bucket this cluster does not own + except ValueError: + continue + + expected_state = (backup_manifest.BackupManifestState.DELETING + if backup.status == Backup.STATUS_DELETING + else backup_manifest.BackupManifestState.ACTIVE) + manifest = backup_manifest.read_if_present(config, UUID(backup.uuid)) + + if manifest is None: + problems += 1 + logger.error("Backup %s (%s) has no manifest in %s", + backup.uuid, backup.status, config.bucket_name) + if repair: + _rewrite_manifest_from_record(backup) + repaired += 1 + continue + + disagreements = [] + if manifest.state != expected_state: + disagreements.append(f"state={manifest.state}, expected {expected_state}") + if manifest.s3_id != backup.s3_id: + disagreements.append(f"s3_id={manifest.s3_id}, expected {backup.s3_id}") + manifest_prev = str(manifest.prev_backup_id) if manifest.prev_backup_id else "" + if manifest_prev != backup.prev_backup_id: + disagreements.append( + f"prev_backup_id={manifest_prev or '(none)'}, " + f"expected {backup.prev_backup_id or '(none)'}") + if not disagreements: + continue + + problems += 1 + logger.error("Backup %s manifest disagrees with its record: %s", + backup.uuid, "; ".join(disagreements)) + if repair: + _rewrite_manifest_from_record(backup) + repaired += 1 + + try: + manifests = discover_backups(config) + except backup_manifest.ManifestError as e: + problems += 1 + logger.error("Could not list bucket %s for cluster %s: %s", + config.bucket_name, cluster.get_id(), e) + continue + + for manifest in manifests: + if manifest.state != backup_manifest.BackupManifestState.ACTIVE: + continue # a delete's own manifest; may briefly outlive its record + if manifest.s3_id not in known_s3_ids: + orphans += 1 + logger.error( + "Manifest for backup %s (s3_id=%s) in %s names no known " + "record in cluster %s", manifest.backup_id, manifest.s3_id, + config.bucket_name, cluster.get_id()) + + logger.info("Backup check: %d problem(s), %d repaired, %d orphaned manifest(s) " + "(never auto-repaired)", problems, repaired, orphans) + if problems > repaired and not repair: + logger.error("Re-run with --repair to rewrite manifests from their records.") + return (problems - repaired) == 0 and orphans == 0 + + +def _rewrite_manifest_from_record(backup: Backup) -> None: + """Rewrite ``backup``'s manifest from its FDB record: ACTIVE, then + DELETING again if that is what the record says. Two steps because + ``write_manifest`` always writes ACTIVE (it is what a freshly completed + backup's manifest has never not been), and only a second, explicit mark + turns that into DELETING -- the same two calls ``_start_backup_delete`` + itself would have made, in the same order. + """ + write_manifest(backup) + if backup.status == Backup.STATUS_DELETING: + mark_manifest_deleting(backup) + + def trigger_merge(keep_backup, old_backup): """Merge ``old_backup`` into ``keep_backup``, its successor in one volume's chain: claim the chain, mark the old one merging, and queue the merge task. diff --git a/simplyblock_core/controllers/backup/manifest.py b/simplyblock_core/controllers/backup/manifest.py index d118ff12ee..be70d760c7 100644 --- a/simplyblock_core/controllers/backup/manifest.py +++ b/simplyblock_core/controllers/backup/manifest.py @@ -470,7 +470,7 @@ def delete(config: BackupConfig, backup_id: UUID) -> None: raise ManifestError(f"Failed to delete manifest for backup {backup_id}") from e -def _read_if_present(config: BackupConfig, backup_id: UUID) -> BackupManifest | None: +def read_if_present(config: BackupConfig, backup_id: UUID) -> BackupManifest | None: """:func:`read`, but ``None`` for a manifest the bucket does not hold.""" try: body = s3_client(config).get_object( @@ -502,7 +502,7 @@ def mark_deleting(config: BackupConfig, backup_id: UUID) -> None: Raises: ManifestError: the manifest could not be read or rewritten. """ - manifest = _read_if_present(config, backup_id) + manifest = read_if_present(config, backup_id) if manifest is not None and manifest.state != BackupManifestState.DELETING: write(config, manifest.model_copy(update={"state": BackupManifestState.DELETING})) @@ -518,7 +518,7 @@ def mark_active(config: BackupConfig, backup_id: UUID) -> None: Raises: ManifestError: the manifest could not be read or rewritten. """ - manifest = _read_if_present(config, backup_id) + manifest = read_if_present(config, backup_id) if manifest is not None and manifest.state != BackupManifestState.ACTIVE: write(config, manifest.model_copy(update={"state": BackupManifestState.ACTIVE})) diff --git a/tests/integration/backup/test_check_backups.py b/tests/integration/backup/test_check_backups.py new file mode 100644 index 0000000000..541cc6cf0f --- /dev/null +++ b/tests/integration/backup/test_check_backups.py @@ -0,0 +1,189 @@ +"""Checking a backup's FDB record against its S3 manifest, against real +FoundationDB. + +The backup analog of the secondary-index checker (``check_indices``): +read-only unless ``--repair`` is given, and reports in both directions -- +a record whose manifest is missing or disagrees, and a manifest naming no +known record. Only the first direction is ever auto-repaired; an orphaned +manifest may be a disaster-recovery backup nothing has imported yet, which +looks identical to genuine drift from here. + +The database is real. The bucket is the one boundary faked, as an in-memory +one, so the manifest functions under test run for real over it. +""" +import io +import zlib +from uuid import UUID + +import pytest +from botocore.exceptions import ClientError + +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import ( + BackupManifest, + BackupManifestState, + ManifestDataPlane, + ManifestSource, + ManifestVolume, +) +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster + + +def _uuid(name: str) -> str: + return f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000" + + +CLUSTER = _uuid("cluster") +OTHER_CLUSTER = _uuid("other-cluster") +LVOL = _uuid("lvol") +NODE = _uuid("node") +BUCKET = "simplyblock-backup-cluster" + + +def _config(): + return BackupConfig.model_validate({"bucket_name": BUCKET, "region": "eu-central-1"}) + + +@pytest.fixture +def db(): + db = DBController() + for cluster_id in (CLUSTER, OTHER_CLUSTER): + c = Cluster() + c.uuid = cluster_id + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return db + + +class _Bucket: + def __init__(self): + self.objects: dict = {} + + def put_object(self, Bucket, Key, Body, **_): + self.objects[Key] = Body + + def get_object(self, Bucket, Key): + if Key not in self.objects: + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") + return {"Body": io.BytesIO(self.objects[Key])} + + def delete_object(self, Bucket, Key): + self.objects.pop(Key, None) + + def get_paginator(self, _name): + objects = self.objects + + class _Paginator: + def paginate(self, Bucket, Prefix): + yield {"Contents": [{"Key": k} for k in objects if k.startswith(Prefix)]} + return _Paginator() + + +@pytest.fixture +def bucket(monkeypatch): + b = _Bucket() + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + return b + + +def _manifest(backup_id: str, prev: str = "", state=BackupManifestState.ACTIVE) -> BackupManifest: + return BackupManifest( + backup_id=UUID(backup_id), + s3_id=zlib.crc32(backup_id.encode()) % 1000, + created_at=100, + completed_at=200, + size=4096, + prev_backup_id=UUID(prev) if prev else None, + source=ManifestSource(cluster_id=UUID(CLUSTER), node_id=UUID(NODE)), + volume=ManifestVolume( + lvol_id=UUID(LVOL), lvol_name="vol", snapshot_id=UUID(_uuid("snap")), + snapshot_name="snap", size=4096), + dataplane=ManifestDataPlane(), + state=state, + ) + + +def _backup(db, name, prev="", status=Backup.STATUS_COMPLETED, cluster_id=CLUSTER): + b = Backup() + b.uuid = _uuid(name) + b.cluster_id = cluster_id + b.lvol_id = LVOL + b.node_id = NODE + b.s3_id = zlib.crc32(name.encode()) % 1000 + b.prev_backup_id = _uuid(prev) if prev else "" + b.status = status + b.location = _config().location().model_dump(mode="json") + b.write_to_db(db.kv_store) + return b + + +def test_a_consistent_completed_backup_passes(db, bucket): + b = _backup(db, "leaf") + backup_manifest.write(_config(), _manifest(b.uuid)) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is True + + +def test_a_completed_backup_with_no_manifest_fails_and_repairs(db, bucket): + b = _backup(db, "leaf") + + assert backup_controller.check_backups(cluster_id=CLUSTER) is False + + assert backup_controller.check_backups(cluster_id=CLUSTER, repair=True) is True + assert backup_manifest.read(_config(), UUID(b.uuid)).state == BackupManifestState.ACTIVE + # A second run finds it already fixed. + assert backup_controller.check_backups(cluster_id=CLUSTER) is True + + +def test_a_disagreeing_manifest_fails_and_repairs(db, bucket): + b = _backup(db, "leaf", prev="base") + # Wrong prev_backup_id in the manifest -- as if a merge updated the + # record but not (yet) the bucket. + backup_manifest.write(_config(), _manifest(b.uuid, prev="")) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is False + + assert backup_controller.check_backups(cluster_id=CLUSTER, repair=True) is True + assert backup_manifest.read(_config(), UUID(b.uuid)).prev_backup_id == UUID(_uuid("base")) + + +def test_a_deleting_backup_with_an_active_manifest_fails_and_repairs(db, bucket): + b = _backup(db, "leaf", status=Backup.STATUS_DELETING) + backup_manifest.write(_config(), _manifest(b.uuid, state=BackupManifestState.ACTIVE)) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is False + + assert backup_controller.check_backups(cluster_id=CLUSTER, repair=True) is True + assert backup_manifest.read(_config(), UUID(b.uuid)).state == BackupManifestState.DELETING + + +def test_an_orphaned_manifest_fails_and_is_never_repaired(db, bucket): + # A manifest with no backing record at all -- s3_id matches nothing. + backup_manifest.write(_config(), _manifest(_uuid("ghost"))) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is False + assert backup_controller.check_backups(cluster_id=CLUSTER, repair=True) is False + + +def test_a_merged_backups_leftover_manifest_is_not_flagged(db, bucket): + """MERGED/MERGING are excluded: the merge path does not mark manifest + state, so a leftover manifest here is a known gap, not this check's to + report.""" + b = _backup(db, "old", status=Backup.STATUS_MERGED) + backup_manifest.write(_config(), _manifest(b.uuid)) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is True + + +def test_scoping_to_one_cluster_ignores_another_clusters_problems(db, bucket): + _backup(db, "leaf", cluster_id=CLUSTER) + backup_manifest.write(_config(), _manifest(_uuid("leaf"))) + # No manifest for this one -- a real problem, but in a different cluster. + _backup(db, "other-leaf", cluster_id=OTHER_CLUSTER) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is True + assert backup_controller.check_backups(cluster_id=OTHER_CLUSTER) is False From 0558631029a294771303a60c251be327a3260a93 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 13 Nov 2026 02:35:41 +0100 Subject: [PATCH 05/13] Fix regressions and test bugs CI's integration run caught CI (Integration tests py3.11/py3.14t) is where the sandbox's broken FDB testcontainer never let this get verified. It found six real problems: Real regressions in the implementation: - delete_backup had no path for a FAILED backup -- it fell through to "not completed, refuse", when delete_backups (the bulk path) already handled this correctly (nothing was ever uploaded, so there is no manifest or S3 data to claim; remove it directly). - group_backup._retire_members treated calling delete_backup on a successor-less member as immediately retiring it, which was true when delete_backup was synchronous and is not now: a member only claimed (DELETING) is not yet gone, and advance_group_backup_delete must keep returning PENDING until it actually is. Fixed by checking the member's real post-call state rather than assuming it. - group_backup._members_touched (abandon_group_backup_delete's "was any member touched" check) didn't count DELETING as touched, so a timed-out delete whose only progress was claiming a member for deletion would have been reported as untouched and the group backup restored to COMPLETED over a member whose S3 objects might still be getting destroyed. Test bugs in this branch's own new test file (test_check_backups.py): - its s3_id and the Backup fixture's s3_id were derived from different strings (the manifest's from the uuid, the record's from a short name), so they never matched -- every manifest looked like an orphan. - the Backup fixture had no snapshot_id, which build_manifest (the --repair path) requires; repairing crashed instead of rewriting. Both are now also handled defensively in check_backups itself: _rewrite_manifest_from_record no longer propagates a build/write failure out of the whole check -- it reports that one backup as unrepaired and keeps going. Legacy stubbed-DB tests (test_legacy.py's own documented pattern, migrated per tests/AGENTS.md -- "convert it to real FDB, don't add a matching patch for the new module"): TestEvaluatePolicy, TestTriggerMerge and TestDeleteBackups.test_success mocked away write_to_db or db_controller entirely, which worked while trigger_merge/delete_backups only mutated in-memory objects and called a no-op write. Now that both read the chain's real siblings through DBController.claim_backup_chain_op, the backups they exercise have to actually be in FDB. Converted to write real records and assert against what the database holds afterward, following the pattern already used in this branch's own test_delete_backup.py. Also updated two test_group_backup_delete.py assertions that depended on delete_backup being synchronous (a leaf member is now DELETING after one retirement pass, not already gone -- its own task removes it, simulated here the same way the existing test already simulated a merge finishing). Lint, mypy, and the full unit suite (6965 tests) are green locally; pushing for CI to re-verify the integration tier, which is what actually caught all of this. --- .../controllers/backup/controller.py | 34 ++++++--- .../controllers/backup/group_backup.py | 45 ++++++++++-- .../integration/backup/test_check_backups.py | 27 ++++--- .../backup/test_group_backup_delete.py | 7 +- tests/integration/backup/test_legacy.py | 72 +++++++++++++------ 5 files changed, 138 insertions(+), 47 deletions(-) diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py index 7e69998621..450b842b25 100644 --- a/simplyblock_core/controllers/backup/controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -1129,6 +1129,15 @@ def delete_backup(backup_id): tasks_controller.add_backup_delete_task(backup.cluster_id, backup.node_id, [backup.uuid]) return + if backup.status == Backup.STATUS_FAILED: + # Uploaded nothing restorable, so there is no manifest and nothing + # in S3 to claim or wait on -- the same direct removal + # delete_backups gives every FAILED backup of a volume. + cleanup_backup_kms_keys([backup]) + backup_events.backup_deleted(backup.cluster_id, backup.node_id, backup) + backup.remove(db_controller.kv_store) + return + dependents = [b for b in db_controller.get_backups_by_lvol_id(backup.lvol_id) if b.prev_backup_id == backup.uuid] if dependents: @@ -1466,8 +1475,7 @@ def check_backups(cluster_id=None, repair=False) -> bool: problems += 1 logger.error("Backup %s (%s) has no manifest in %s", backup.uuid, backup.status, config.bucket_name) - if repair: - _rewrite_manifest_from_record(backup) + if repair and _rewrite_manifest_from_record(backup): repaired += 1 continue @@ -1487,8 +1495,7 @@ def check_backups(cluster_id=None, repair=False) -> bool: problems += 1 logger.error("Backup %s manifest disagrees with its record: %s", backup.uuid, "; ".join(disagreements)) - if repair: - _rewrite_manifest_from_record(backup) + if repair and _rewrite_manifest_from_record(backup): repaired += 1 try: @@ -1516,17 +1523,28 @@ def check_backups(cluster_id=None, repair=False) -> bool: return (problems - repaired) == 0 and orphans == 0 -def _rewrite_manifest_from_record(backup: Backup) -> None: +def _rewrite_manifest_from_record(backup: Backup) -> bool: """Rewrite ``backup``'s manifest from its FDB record: ACTIVE, then DELETING again if that is what the record says. Two steps because ``write_manifest`` always writes ACTIVE (it is what a freshly completed backup's manifest has never not been), and only a second, explicit mark turns that into DELETING -- the same two calls ``_start_backup_delete`` itself would have made, in the same order. + + Never raises: a record this cannot rebuild from (an unreadable location, + a missing encryption descriptor, an unreachable bucket) is one + ``check_backups`` has to report and move on from, not one that aborts + the whole run over a single bad backup. Returns whether it succeeded. """ - write_manifest(backup) - if backup.status == Backup.STATUS_DELETING: - mark_manifest_deleting(backup) + try: + write_manifest(backup) + if backup.status == Backup.STATUS_DELETING: + mark_manifest_deleting(backup) + except (ValueError, backup_manifest.ManifestError, PreconditionError, KeyError) as e: + logger.error("Could not rewrite the manifest of backup %s from its " + "record: %s", backup.uuid, e) + return False + return True def trigger_merge(keep_backup, old_backup): diff --git a/simplyblock_core/controllers/backup/group_backup.py b/simplyblock_core/controllers/backup/group_backup.py index 695b4e55aa..6912d0dae6 100644 --- a/simplyblock_core/controllers/backup/group_backup.py +++ b/simplyblock_core/controllers/backup/group_backup.py @@ -624,13 +624,19 @@ def _config_for_deleting(db, group_backup): def _members_touched(db, group_backup) -> bool: - """Whether retiring a member has begun: a member deleted, merging, or merged.""" + """Whether retiring a member has begun: a member deleted, merging, + merged, or claimed for deletion (DELETING -- not yet actually gone, but + its own FN_BACKUP_DELETE task may already be destroying its S3 objects, + which is exactly the in-flight state a restore back to COMPLETED must + not paper over). + """ for backup_id in group_backup.member_backup_ids: try: member = db.get_backup_by_id(backup_id) except KeyError: return True - if member.status in (Backup.STATUS_MERGING, Backup.STATUS_MERGED): + if member.status in (Backup.STATUS_MERGING, Backup.STATUS_MERGED, + Backup.STATUS_DELETING): return True return False @@ -652,6 +658,13 @@ def _retire_members(db, group_backup) -> bool: continue # deleted if member.status == Backup.STATUS_MERGED: continue + if member.status == Backup.STATUS_DELETING: + # Already claimed by an earlier pass; its own FN_BACKUP_DELETE + # task is retiring it. Not retired until that task actually + # removes the record (the KeyError branch above, on a later + # pass) -- re-claiming it here would needlessly re-queue the task. + retired = False + continue chain = db.get_backups_by_lvol_id(member.lvol_id) successor = next((b for b in chain if b.prev_backup_id == member.uuid), None) busy = (member.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS) @@ -661,16 +674,34 @@ def _retire_members(db, group_backup) -> bool: backup_controller.delete_backup(member.uuid) except KeyError: pass # deleted concurrently - logger.info("Group backup %s: member %s deleted, nothing chains against it", - group_backup.uuid, member.uuid[:8]) + # A FAILED member is removed synchronously (nothing was ever + # uploaded for it, so there is no manifest or task to wait on); + # a COMPLETED one is only claimed -- DELETING now, actually gone + # once its own FN_BACKUP_DELETE task finishes. Checked rather + # than branched on the member's status before the call, so this + # reads the one fact that actually answers it either way. + try: + db.get_backup_by_id(member.uuid) + except KeyError: + logger.info("Group backup %s: member %s deleted, nothing " + "chains against it", group_backup.uuid, member.uuid[:8]) + else: + logger.info("Group backup %s: member %s claimed for deletion, " + "nothing chains against it", + group_backup.uuid, member.uuid[:8]) + retired = False continue retired = False if (successor is not None and not busy and member.status == Backup.STATUS_COMPLETED and successor.status == Backup.STATUS_COMPLETED): - backup_controller.trigger_merge(successor, member) - logger.info("Group backup %s: member %s merging into its successor %s", - group_backup.uuid, member.uuid[:8], successor.uuid[:8]) + ok, reason = backup_controller.trigger_merge(successor, member) + if ok: + logger.info("Group backup %s: member %s merging into its successor %s", + group_backup.uuid, member.uuid[:8], successor.uuid[:8]) + else: + logger.info("Group backup %s: member %s not merged into %s yet: %s", + group_backup.uuid, member.uuid[:8], successor.uuid[:8], reason) return retired diff --git a/tests/integration/backup/test_check_backups.py b/tests/integration/backup/test_check_backups.py index 541cc6cf0f..56d038d5c1 100644 --- a/tests/integration/backup/test_check_backups.py +++ b/tests/integration/backup/test_check_backups.py @@ -90,10 +90,15 @@ def bucket(monkeypatch): return b -def _manifest(backup_id: str, prev: str = "", state=BackupManifestState.ACTIVE) -> BackupManifest: +def _manifest(backup_id: str, prev: str = "", state=BackupManifestState.ACTIVE, + s3_id: int | None = None) -> BackupManifest: + # s3_id defaults to a value derived from backup_id itself -- fine for an + # orphan manifest naming no real backup, but a manifest meant to describe + # a real one MUST pass that backup's own s3_id: a Backup's s3_id is + # whatever was assigned at creation, not derivable from its uuid. return BackupManifest( backup_id=UUID(backup_id), - s3_id=zlib.crc32(backup_id.encode()) % 1000, + s3_id=s3_id if s3_id is not None else zlib.crc32(backup_id.encode()) % 1000, created_at=100, completed_at=200, size=4096, @@ -112,7 +117,13 @@ def _backup(db, name, prev="", status=Backup.STATUS_COMPLETED, cluster_id=CLUSTE b.uuid = _uuid(name) b.cluster_id = cluster_id b.lvol_id = LVOL + b.lvol_name = "vol" + # build_manifest (the --repair path) needs a valid snapshot_id: a real + # COMPLETED backup always has one. + b.snapshot_id = _uuid(f"snap-{name}") + b.snapshot_name = "snap" b.node_id = NODE + b.size = 4096 b.s3_id = zlib.crc32(name.encode()) % 1000 b.prev_backup_id = _uuid(prev) if prev else "" b.status = status @@ -123,7 +134,7 @@ def _backup(db, name, prev="", status=Backup.STATUS_COMPLETED, cluster_id=CLUSTE def test_a_consistent_completed_backup_passes(db, bucket): b = _backup(db, "leaf") - backup_manifest.write(_config(), _manifest(b.uuid)) + backup_manifest.write(_config(), _manifest(b.uuid, s3_id=b.s3_id)) assert backup_controller.check_backups(cluster_id=CLUSTER) is True @@ -143,7 +154,7 @@ def test_a_disagreeing_manifest_fails_and_repairs(db, bucket): b = _backup(db, "leaf", prev="base") # Wrong prev_backup_id in the manifest -- as if a merge updated the # record but not (yet) the bucket. - backup_manifest.write(_config(), _manifest(b.uuid, prev="")) + backup_manifest.write(_config(), _manifest(b.uuid, prev="", s3_id=b.s3_id)) assert backup_controller.check_backups(cluster_id=CLUSTER) is False @@ -153,7 +164,7 @@ def test_a_disagreeing_manifest_fails_and_repairs(db, bucket): def test_a_deleting_backup_with_an_active_manifest_fails_and_repairs(db, bucket): b = _backup(db, "leaf", status=Backup.STATUS_DELETING) - backup_manifest.write(_config(), _manifest(b.uuid, state=BackupManifestState.ACTIVE)) + backup_manifest.write(_config(), _manifest(b.uuid, state=BackupManifestState.ACTIVE, s3_id=b.s3_id)) assert backup_controller.check_backups(cluster_id=CLUSTER) is False @@ -174,14 +185,14 @@ def test_a_merged_backups_leftover_manifest_is_not_flagged(db, bucket): state, so a leftover manifest here is a known gap, not this check's to report.""" b = _backup(db, "old", status=Backup.STATUS_MERGED) - backup_manifest.write(_config(), _manifest(b.uuid)) + backup_manifest.write(_config(), _manifest(b.uuid, s3_id=b.s3_id)) assert backup_controller.check_backups(cluster_id=CLUSTER) is True def test_scoping_to_one_cluster_ignores_another_clusters_problems(db, bucket): - _backup(db, "leaf", cluster_id=CLUSTER) - backup_manifest.write(_config(), _manifest(_uuid("leaf"))) + b = _backup(db, "leaf", cluster_id=CLUSTER) + backup_manifest.write(_config(), _manifest(b.uuid, s3_id=b.s3_id)) # No manifest for this one -- a real problem, but in a different cluster. _backup(db, "other-leaf", cluster_id=OTHER_CLUSTER) diff --git a/tests/integration/backup/test_group_backup_delete.py b/tests/integration/backup/test_group_backup_delete.py index c0eb99a7b0..4a9abcca98 100644 --- a/tests/integration/backup/test_group_backup_delete.py +++ b/tests/integration/backup/test_group_backup_delete.py @@ -218,7 +218,9 @@ def test_a_leaf_is_deleted_and_a_base_merged_while_the_group_stays(self, db, buc state = group_backup.advance_group_backup_delete(gb.get_id()) assert state == group_backup.DeleteState.PENDING - assert _status(db, leaf) is None + # Claimed, not yet actually gone: leaf's own FN_BACKUP_DELETE task + # removes it once it confirms the S3 objects are gone. + assert _status(db, leaf) == Backup.STATUS_DELETING assert _status(db, base) == Backup.STATUS_MERGING assert db.get_group_backup_by_id(gb.get_id()).status == GroupBackup.STATUS_DELETING assert bucket[GEN_7].state == GroupManifestState.DELETING @@ -230,6 +232,9 @@ def test_the_manifest_and_record_go_only_once_every_member_is_retired(self, db, merged = db.get_backup_by_id(base.get_id()) merged.status = Backup.STATUS_MERGED # the merge task finished merged.write_to_db(db.kv_store) + # leaf's own FN_BACKUP_DELETE task finished too: it confirmed the S3 + # objects were gone and removed the record. + db.get_backup_by_id(leaf.get_id()).remove(db.kv_store) state = group_backup.advance_group_backup_delete(gb.get_id()) diff --git a/tests/integration/backup/test_legacy.py b/tests/integration/backup/test_legacy.py index 9f3a15e208..bbe906c057 100644 --- a/tests/integration/backup/test_legacy.py +++ b/tests/integration/backup/test_legacy.py @@ -743,22 +743,28 @@ def test_incomplete_chain_is_refused(self): class TestDeleteBackups(unittest.TestCase): - @patch("simplyblock_core.controllers.backup.controller.backup_events") - @patch("simplyblock_core.models.storage_node.RPCClient") - @patch("simplyblock_core.controllers.backup.controller.db_controller") - def test_success(self, mock_db, MockRPC, mock_events): + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") + def test_success(self, mock_boto3_client): + # delete_backups claims the chain and marks the manifest DELETING + # against the real database and a real (faked) S3 client -- it no + # longer removes the record synchronously, so there is nothing left + # here to mock away the way the old, synchronous delete_backups did. + from botocore.exceptions import ClientError + mock_boto3_client.return_value.get_object.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey"}}, "GetObject") + + db = DBController() + cluster = _cluster() + cluster.write_to_db(db.kv_store) b1 = _backup(uuid="b-1") - mock_db.get_backups_by_lvol_id.return_value = [b1] - mock_db.get_storage_node_by_id.return_value = _node() - b1.remove = MagicMock() - MockRPC.return_value.bdev_lvol_s3_delete.return_value = True + b1.location = _backup_config().location().model_dump(mode="json") + b1.write_to_db(db.kv_store) from simplyblock_core.controllers.backup.controller import delete_backups success, error = delete_backups("lvol-1") - self.assertTrue(success) - self.assertIsNone(error) - b1.remove.assert_called_once() + self.assertTrue(success, error) + self.assertEqual(db.get_backup_by_id("b-1").status, Backup.STATUS_DELETING) @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_no_backups(self, mock_db): @@ -984,17 +990,24 @@ def test_no_policy(self, mock_db, mock_tasks): mock_tasks.add_backup_merge_task.assert_not_called() - @patch.object(Backup, 'write_to_db') @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") - def test_version_limit_exceeded(self, mock_db, mock_tasks, _mock_write): + def test_version_limit_exceeded(self, mock_db, mock_tasks): policy = _policy(max_versions=2, max_age_seconds=0) mock_db.get_policy_for_lvol.return_value = policy + # trigger_merge's own chain guard (claim_backup_chain_op) reads the + # REAL database, not policy.py's mocked one above -- see the module + # docstring on why this file is legacy and tests/AGENTS.md on why a + # patched DB only covers the module it names. The backups the guard + # has to find therefore have to be real. + db = DBController() now = int(time.time()) b1 = _backup(uuid="b1", created_at=now - 300) b2 = _backup(uuid="b2", created_at=now - 200) b3 = _backup(uuid="b3", created_at=now - 100) + for b in (b1, b2, b3): + b.write_to_db(db.kv_store) mock_db.get_backups_by_lvol_id.return_value = [b1, b2, b3] from simplyblock_core.controllers.backup.policy import evaluate_policy @@ -1020,17 +1033,19 @@ def test_under_version_limit(self, mock_db, mock_tasks): mock_tasks.add_backup_merge_task.assert_not_called() - @patch.object(Backup, 'write_to_db') @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") - def test_age_limit_exceeded(self, mock_db, mock_tasks, _mock_write): + def test_age_limit_exceeded(self, mock_db, mock_tasks): policy = _policy(max_versions=0) policy.max_age_seconds = 3600 # 1 hour mock_db.get_policy_for_lvol.return_value = policy + db = DBController() now = int(time.time()) b1 = _backup(uuid="b1", created_at=now - 7200) # 2 hours old b2 = _backup(uuid="b2", created_at=now - 100) + for b in (b1, b2): + b.write_to_db(db.kv_store) mock_db.get_backups_by_lvol_id.return_value = [b1, b2] from simplyblock_core.controllers.backup.policy import evaluate_policy @@ -1039,18 +1054,20 @@ def test_age_limit_exceeded(self, mock_db, mock_tasks, _mock_write): mock_tasks.add_backup_merge_task.assert_called_once() - @patch.object(Backup, 'write_to_db') @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") - def test_both_conditions_required(self, mock_db, mock_tasks, _mock_write): + def test_both_conditions_required(self, mock_db, mock_tasks): """When both versions and age are set, either limit can trigger a merge.""" policy = _policy(max_versions=3) policy.max_age_seconds = 3600 mock_db.get_policy_for_lvol.return_value = policy + db = DBController() now = int(time.time()) # 4 backups (exceeds version limit of 3) but all recent (doesn't exceed age) backups = [_backup(uuid=f"b{i}", created_at=now - (i * 60)) for i in range(4)] + for b in backups: + b.write_to_db(db.kv_store) mock_db.get_backups_by_lvol_id.return_value = backups from simplyblock_core.controllers.backup.policy import evaluate_policy @@ -1059,19 +1076,21 @@ def test_both_conditions_required(self, mock_db, mock_tasks, _mock_write): mock_tasks.add_backup_merge_task.assert_called_once() - @patch.object(Backup, 'write_to_db') @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") - def test_both_conditions_met(self, mock_db, mock_tasks, _mock_write): + def test_both_conditions_met(self, mock_db, mock_tasks): """When both limits set and both exceeded, merge triggers.""" policy = _policy(max_versions=2) policy.max_age_seconds = 3600 mock_db.get_policy_for_lvol.return_value = policy + db = DBController() now = int(time.time()) b1 = _backup(uuid="b1", created_at=now - 7200) b2 = _backup(uuid="b2", created_at=now - 200) b3 = _backup(uuid="b3", created_at=now - 100) + for b in (b1, b2, b3): + b.write_to_db(db.kv_store) mock_db.get_backups_by_lvol_id.return_value = [b1, b2, b3] from simplyblock_core.controllers.backup.policy import evaluate_policy @@ -1295,14 +1314,21 @@ class TestTriggerMerge(unittest.TestCase): @patch("simplyblock_core.controllers.backup.controller.tasks_controller") def test_trigger_merge_marks_old_as_merging(self, mock_tasks): from simplyblock_core.controllers.backup.controller import trigger_merge + # trigger_merge's chain guard (claim_backup_chain_op) flips the status + # of the record it finds in the REAL database, inside its own + # transaction -- never the caller's in-memory object -- so both + # backups have to be real, and the result is read back through the + # database too. + db = DBController() keep = _backup(uuid="keep") old = _backup(uuid="old") - old.write_to_db = MagicMock() + keep.write_to_db(db.kv_store) + old.write_to_db(db.kv_store) - trigger_merge(keep, old) + ok, reason = trigger_merge(keep, old) - self.assertEqual(old.status, Backup.STATUS_MERGING) - old.write_to_db.assert_called_once() + self.assertTrue(ok, reason) + self.assertEqual(db.get_backup_by_id("old").status, Backup.STATUS_MERGING) mock_tasks.add_backup_merge_task.assert_called_once() @patch("simplyblock_core.controllers.backup.controller.tasks_controller") From a291d3b50552689652f3602147eac9726264d34d Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 13 Nov 2026 03:00:46 +0100 Subject: [PATCH 06/13] Fix TestDeleteBackups.test_success: use a real UUID for the backup id Its backup used the short placeholder "b-1", which every other test in this file gets away with since nothing used to parse it -- but delete_backups now reaches manifest.mark_deleting, which does UUID(backup.uuid) (via manifest_key / build_manifest's backup_id). "b-1" is not shaped like one. CI's last run (py3.11 and py3.14t integration, both green otherwise) was the only thing still failing. --- tests/integration/backup/test_legacy.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/integration/backup/test_legacy.py b/tests/integration/backup/test_legacy.py index bbe906c057..06c4ecaa6e 100644 --- a/tests/integration/backup/test_legacy.py +++ b/tests/integration/backup/test_legacy.py @@ -756,7 +756,12 @@ def test_success(self, mock_boto3_client): db = DBController() cluster = _cluster() cluster.write_to_db(db.kv_store) - b1 = _backup(uuid="b-1") + # A real UUID, not the short "b-1" placeholder used elsewhere in this + # file: the manifest path now parses backup.uuid with UUID(...) + # (manifest_key, build_manifest's backup_id), which a short string + # like "b-1" is not shaped like. + backup_uuid = "00000000-0000-4000-8000-000000000001" + b1 = _backup(uuid=backup_uuid) b1.location = _backup_config().location().model_dump(mode="json") b1.write_to_db(db.kv_store) @@ -764,7 +769,7 @@ def test_success(self, mock_boto3_client): success, error = delete_backups("lvol-1") self.assertTrue(success, error) - self.assertEqual(db.get_backup_by_id("b-1").status, Backup.STATUS_DELETING) + self.assertEqual(db.get_backup_by_id(backup_uuid).status, Backup.STATUS_DELETING) @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_no_backups(self, mock_db): From fccb9839f351429e9b034a7d7258ffd38bb5b7ee Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 13 Nov 2026 14:32:22 +0100 Subject: [PATCH 07/13] Fix backup deletion: there is no data-plane RPC to remove S3 objects bdev_lvol_s3_delete does not exist on the data plane and never will -- confirmed live, it always answers Method not found. A tip or whole-chain delete needs no data-plane involvement at all: the control plane can remove a backup's S3 objects itself, the same way it already removes a manifest. Where the data plane genuinely has to be involved -- a backup something still chains against -- that already goes through bdev_lvol_s3_merge instead, which stays unchanged. Adds backup_manifest.delete_data_objects (list+batch-delete under the backup's s3_id prefix) and backup_controller.delete_backup_data, and switches tasks_runner_backup._run_delete to call it instead of the RPC. Removes the now-dead RPCClient.bdev_lvol_s3_delete method and the test asserting it exists, and fixes a db_controller.py comment that referenced the same nonexistent RPC. --- .../controllers/backup/controller.py | 19 +++++ .../controllers/backup/manifest.py | 40 +++++++++ simplyblock_core/db_controller.py | 7 +- simplyblock_core/rpc_client.py | 5 -- .../services/tasks_runner_backup.py | 27 +++--- .../integration/backup/test_delete_backup.py | 84 ++++++++++++++++++- tests/integration/backup/test_legacy.py | 9 +- 7 files changed, 165 insertions(+), 26 deletions(-) diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py index 450b842b25..37383559b7 100644 --- a/simplyblock_core/controllers/backup/controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -377,6 +377,25 @@ def delete_manifest(backup: Backup) -> None: backup_manifest.delete(_config_for(backup), UUID(backup.uuid)) +def delete_backup_data(backup: Backup) -> None: + """Remove a backup's S3 data objects directly, with no data-plane RPC. + + There is no data-plane RPC for this and none is planned -- a bucket's data + objects can be removed the same way a manifest is, by the control plane + talking to S3 itself. This is only for a backup whose delete needs no + data-plane involvement (a tip delete, or a whole-chain delete): a backup + something still chains against is folded into its successor by + ``trigger_merge`` instead, which is where the data plane's own + ``bdev_lvol_s3_merge`` remains the only way to remove such a backup's + objects without breaking its descendant. + + Raises: + ManifestError: the objects could not be listed or removed. + PreconditionError: the backup's bucket is not the cluster's own. + """ + backup_manifest.delete_data_objects(_config_for(backup), backup.s3_id) + + def mark_manifest_deleting(backup: Backup) -> None: """Mark a backup's manifest DELETING, before any of its S3 objects are touched. See ``manifest.mark_deleting``. diff --git a/simplyblock_core/controllers/backup/manifest.py b/simplyblock_core/controllers/backup/manifest.py index be70d760c7..146d652652 100644 --- a/simplyblock_core/controllers/backup/manifest.py +++ b/simplyblock_core/controllers/backup/manifest.py @@ -470,6 +470,46 @@ def delete(config: BackupConfig, backup_id: UUID) -> None: raise ManifestError(f"Failed to delete manifest for backup {backup_id}") from e +def delete_data_objects(config: BackupConfig, s3_id: int) -> None: + """Remove every object a backup's ``s3_id`` owns under ``key_format`` + (``{s3_id}/{mid}/{extent}``) -- its data clusters and its metadata object. + + Only for a backup whose deletion needs no data-plane involvement: a tip + delete or a whole-chain delete, where nothing else names these objects. + A backup something still chains against is folded into its successor by + the data plane instead (``bdev_lvol_s3_merge``), which removes the old + backup's objects itself as part of the merge -- this function is never + called for that case. + + Idempotent: an ``s3_id`` prefix with nothing left under it lists empty and + this is a no-op, so retrying after a partial failure costs one empty list + call and nothing else. + + Raises: + ManifestError: the bucket could not be listed, or an object under this + prefix could not be removed. + """ + client = s3_client(config) + prefix = f"{s3_id}/" + try: + pages = client.get_paginator("list_objects_v2").paginate( + Bucket=config.bucket_name, Prefix=prefix) + for page in pages: + keys = [{"Key": entry["Key"]} for entry in page.get("Contents", [])] + if not keys: + continue + resp = client.delete_objects( + Bucket=config.bucket_name, Delete={"Objects": keys, "Quiet": True}) + errors = resp.get("Errors") or [] + if errors: + raise ManifestError( + f"Failed to delete {len(errors)} object(s) under {prefix} in " + f"{config.bucket_name}: {errors[0].get('Message')}") + except (BotoCoreError, ClientError) as e: + raise ManifestError( + f"Failed to delete data objects under {prefix} in {config.bucket_name}") from e + + def read_if_present(config: BackupConfig, backup_id: UUID) -> BackupManifest | None: """:func:`read`, but ``None`` for a manifest the bucket does not hold.""" try: diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index cf8e023c25..706d89255b 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -2248,9 +2248,10 @@ def next_vuid(self) -> int: # old allocator was max-plus-one over the local cluster's Backup records, # which had three problems: it raced (two concurrent backups got the same # id), it recycled the id of a deleted backup whose objects may still exist - # (nothing reclaims them -- bdev_lvol_s3_delete does not exist on the data - # plane), and after an import it counted foreign backups it should not have. - # A monotonic sequence makes reuse impossible by construction. + # (deletion removes them, but not atomically with the record disappearing, + # so a window exists where the old id's objects are still reachable), and + # after an import it counted foreign backups it should not have. A + # monotonic sequence makes reuse impossible by construction. # # Unlike vuid this space is NOT unbounded: the data plane packs s3_id into # 30 bits and masks rather than validates, so callers must check diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 467434496f..35252a919d 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -2784,10 +2784,5 @@ def bdev_s3_delete(self, name: str): """ return self._request("bdev_s3_delete", name=name) - def bdev_lvol_s3_delete(self, s3_ids: list[int]): - """Delete all S3 backups for the given IDs (list of uint32).""" - # RPC still missing on data plane — use dummy - return self._request("bdev_lvol_s3_delete", s3_ids=s3_ids) - def bdev_nvme_get_controller_health_info(self, name): return self._request("bdev_nvme_get_controller_health_info", name=name) diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index 09c1f03dc3..5a09cee06f 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -424,9 +424,13 @@ def _run_delete(task): of this same task already retired it, or (the group-backup path) a different caller did. - Single-cycle, unlike backup/restore/merge: ``bdev_lvol_s3_delete`` has no - paired stat RPC to poll in ``rpc_client.py``, so it is treated as - synchronous, exactly as its one existing client method already models it. + No data-plane RPC: there is no ``bdev_lvol_s3_delete`` and none is + planned, so the control plane removes the objects itself, the same way it + removes a manifest. This handler is only ever reached for a backup nothing + chains against -- one with a successor is folded into it by + ``_run_merge`` instead, which is where the data plane stays involved + (``bdev_lvol_s3_merge``). Single-cycle like the RPC path it replaces: S3 + delete is synchronous from the caller's side, with nothing to poll. """ backup_ids = task.function_params.get("backup_ids") or [] if not backup_ids: @@ -446,16 +450,11 @@ def _run_delete(task): set_result(task, "Nothing left to delete") return - snode = _online_node(backups[0].node_id) - rpc_client = snode.rpc_client(timeout=30) - - try: - ret = rpc_client.bdev_lvol_s3_delete([b.s3_id for b in backups]) - except RPCException as e: - _defer_if_busy(e) - raise TaskRetry(f"RPC error: {e}") - if not ret: - raise TaskRetry("bdev_lvol_s3_delete RPC failed") + for backup in backups: + try: + backup_controller.delete_backup_data(backup) + except (ManifestError, PreconditionError) as e: + raise TaskRetry(f"S3 data delete failed: {e}") # The data is confirmed gone: metadata follows data on the way out, the # reverse of how the claim marked intent before any of this ran. @@ -623,7 +622,7 @@ def finalize_resource(task): elif task.function_name == JobSchedule.FN_BACKUP_DELETE: # The delete did not finish (timeout, retry ceiling, abort, # cancellation): every backup this task still finds DELETING had its - # S3 objects left untouched -- the RPC either never ran or never + # S3 objects left untouched -- the delete either never ran or never # confirmed success, and _run_delete only removes a manifest/record # after it has -- so each is reverted to COMPLETED, record and # manifest both, the same way an unfinished merge leaves its old diff --git a/tests/integration/backup/test_delete_backup.py b/tests/integration/backup/test_delete_backup.py index 661fc4bce2..80010db566 100644 --- a/tests/integration/backup/test_delete_backup.py +++ b/tests/integration/backup/test_delete_backup.py @@ -63,8 +63,8 @@ def db(): class _Bucket: - """An in-memory S3 bucket, behind the three object calls the manifest - functions make.""" + """An in-memory S3 bucket, behind the manifest calls and the data-object + listing/deletion calls ``backup_manifest.delete_data_objects`` makes.""" def __init__(self): self.objects: dict = {} @@ -80,6 +80,19 @@ def get_object(self, Bucket, Key): def delete_object(self, Bucket, Key): self.objects.pop(Key, None) + def get_paginator(self, _name): + objects = self.objects + + class _Paginator: + def paginate(self, Bucket, Prefix): + yield {"Contents": [{"Key": k} for k in objects if k.startswith(Prefix)]} + return _Paginator() + + def delete_objects(self, Bucket, Delete): + for entry in Delete["Objects"]: + self.objects.pop(entry["Key"], None) + return {} + @pytest.fixture def bucket(monkeypatch): @@ -337,3 +350,70 @@ def test_delete_backups_is_a_noop_resume_once_everything_is_already_deleting(db, ok, err = backup_controller.delete_backups(LVOL) assert ok, err + + +def _seed_data_objects(bucket, backup): + """Fake data the data plane would have written under this backup's + ``s3_id`` -- a metadata object (``mid=1``) and one data cluster + (``mid=0``), per ``ManifestDataPlane.key_format``.""" + bucket.objects[f"{backup.s3_id}/1/0"] = b"metadata" + bucket.objects[f"{backup.s3_id}/0/0"] = b"cluster-0" + + +class TestDeleteTask: + """``FN_BACKUP_DELETE`` (``tasks_runner_backup._run_delete``): the task + ``backup_controller.delete_backup``/``delete_backups`` queue after + claiming a backup, which removes its S3 objects itself -- there is no + data-plane RPC for this and none is planned. A backup with a successor is + never reached here: that goes through a merge instead + (``trigger_merge``/``_run_merge``), which is where the data plane stays + involved. + """ + + def _task(self, db, backup_id): + return _delete_tasks(db, backup_id)[0] + + def test_deleting_removes_data_objects_manifest_and_record(self, db, bucket): + from simplyblock_core.services import tasks_runner_backup as runner + leaf = _backup(db, bucket, "leaf") + _seed_data_objects(bucket, leaf) + backup_controller.delete_backup(leaf.uuid) + + runner._run_delete(self._task(db, leaf.uuid)) + + assert not any(k.startswith(f"{leaf.s3_id}/") for k in bucket.objects) + assert backup_manifest.manifest_key(UUID(leaf.uuid)) not in bucket.objects + assert not _exists(db, leaf) + + def test_a_backup_no_longer_deleting_is_skipped(self, db, bucket): + """A prior cycle of this same task (or, for a group member, a + different caller) already retired it -- resuming must not re-delete + objects out from under whatever claimed it next.""" + from simplyblock_core.services import tasks_runner_backup as runner + leaf = _backup(db, bucket, "leaf") + backup_controller.delete_backup(leaf.uuid) + task = self._task(db, leaf.uuid) + db.atomic_update(leaf, lambda b: setattr(b, "status", Backup.STATUS_COMPLETED)) + + runner._run_delete(task) + + assert _status(db, leaf) == Backup.STATUS_COMPLETED + + def test_a_bucket_failure_retries_without_touching_the_manifest_or_record( + self, db, bucket, monkeypatch): + from simplyblock_core.services import tasks_runner_backup as runner + from simplyblock_core.services.task_runner_base import TaskRetry + leaf = _backup(db, bucket, "leaf") + _seed_data_objects(bucket, leaf) + backup_controller.delete_backup(leaf.uuid) + + def unreachable(_name): + raise ClientError({"Error": {"Code": "500"}}, "ListObjectsV2") + monkeypatch.setattr(bucket, "get_paginator", unreachable) + + with pytest.raises(TaskRetry): + runner._run_delete(self._task(db, leaf.uuid)) + + assert _status(db, leaf) == Backup.STATUS_DELETING + assert _manifest_state(bucket, leaf) == BackupManifestState.DELETING + assert any(k.startswith(f"{leaf.s3_id}/") for k in bucket.objects) diff --git a/tests/integration/backup/test_legacy.py b/tests/integration/backup/test_legacy.py index 06c4ecaa6e..6dd8bd67f2 100644 --- a/tests/integration/backup/test_legacy.py +++ b/tests/integration/backup/test_legacy.py @@ -1203,9 +1203,14 @@ def test_bdev_lvol_s3_recovery_exists(self): from simplyblock_core.rpc_client import RPCClient self.assertTrue(hasattr(RPCClient, 'bdev_lvol_s3_recovery')) - def test_bdev_lvol_s3_delete_exists(self): + def test_bdev_lvol_s3_delete_is_gone(self): + """There is no data-plane RPC for deleting a backup's S3 objects, and + none is planned -- the control plane removes them itself + (backup_manifest.delete_data_objects), the same way it removes a + manifest. A backup that still has a successor goes through + bdev_lvol_s3_merge instead.""" from simplyblock_core.rpc_client import RPCClient - self.assertTrue(hasattr(RPCClient, 'bdev_lvol_s3_delete')) + self.assertFalse(hasattr(RPCClient, 'bdev_lvol_s3_delete')) # =========================================================================== From 0722f9b3eb502f182d21efc7136b42067e0d7dcb Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 13 Nov 2026 17:28:01 +0100 Subject: [PATCH 08/13] Fix backup-data delete: use per-key delete_object, not batch delete_objects Found live, against a real MinIO backend: delete_objects (multi-object delete) requires a legacy Content-MD5 header that botocore 1.36+ no longer computes by default in favor of its newer flexible checksums -- MinIO (RELEASE.2024-01-16) doesn't understand those and answers MissingContentMD5, and setting request_checksum_calculation back to when_required doesn't restore the header for this operation either. Single-object delete has no such requirement, so delete_data_objects now calls delete_object once per key instead of batching. --- .../controllers/backup/manifest.py | 22 ++++++++++--------- .../integration/backup/test_delete_backup.py | 5 ----- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/simplyblock_core/controllers/backup/manifest.py b/simplyblock_core/controllers/backup/manifest.py index 146d652652..84fdb9e34d 100644 --- a/simplyblock_core/controllers/backup/manifest.py +++ b/simplyblock_core/controllers/backup/manifest.py @@ -367,6 +367,7 @@ def s3_client(config: BackupConfig): Credentials are passed only when configured; omitting them lets boto3 fall back to its default provider chain (instance IAM role, environment, profile), which is what an absent ``credentials`` means. + """ return boto3.client("s3", region_name=config.region, @@ -485,6 +486,15 @@ def delete_data_objects(config: BackupConfig, s3_id: int) -> None: this is a no-op, so retrying after a partial failure costs one empty list call and nothing else. + One ``delete_object`` per key rather than batching through + ``delete_objects``: that call's multi-object delete requires a legacy + Content-MD5 header botocore (1.36+) no longer computes by default, in + favour of the newer flexible checksums -- confirmed live against MinIO + (RELEASE.2024-01-16), which does not understand those and answers + ``MissingContentMD5``, including with ``request_checksum_calculation`` + set back to ``"when_required"``. Single-object delete carries no such + requirement and needs no workaround. + Raises: ManifestError: the bucket could not be listed, or an object under this prefix could not be removed. @@ -495,16 +505,8 @@ def delete_data_objects(config: BackupConfig, s3_id: int) -> None: pages = client.get_paginator("list_objects_v2").paginate( Bucket=config.bucket_name, Prefix=prefix) for page in pages: - keys = [{"Key": entry["Key"]} for entry in page.get("Contents", [])] - if not keys: - continue - resp = client.delete_objects( - Bucket=config.bucket_name, Delete={"Objects": keys, "Quiet": True}) - errors = resp.get("Errors") or [] - if errors: - raise ManifestError( - f"Failed to delete {len(errors)} object(s) under {prefix} in " - f"{config.bucket_name}: {errors[0].get('Message')}") + for entry in page.get("Contents", []): + client.delete_object(Bucket=config.bucket_name, Key=entry["Key"]) except (BotoCoreError, ClientError) as e: raise ManifestError( f"Failed to delete data objects under {prefix} in {config.bucket_name}") from e diff --git a/tests/integration/backup/test_delete_backup.py b/tests/integration/backup/test_delete_backup.py index 80010db566..78a94946ac 100644 --- a/tests/integration/backup/test_delete_backup.py +++ b/tests/integration/backup/test_delete_backup.py @@ -88,11 +88,6 @@ def paginate(self, Bucket, Prefix): yield {"Contents": [{"Key": k} for k in objects if k.startswith(Prefix)]} return _Paginator() - def delete_objects(self, Bucket, Delete): - for entry in Delete["Objects"]: - self.objects.pop(entry["Key"], None) - return {} - @pytest.fixture def bucket(monkeypatch): From c38d71cb10206b199fad1a0f64274a3d715b6d3c Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 18 Nov 2026 16:45:13 +0100 Subject: [PATCH 09/13] Fix silent fork mis-handling in backup chain successor lookups delete_backup, group backup member retirement, and orphan retirement each picked an arbitrary successor via next()/dict construction when a backup had multiple dependents, silently dropping any sibling fork instead of failing loudly. Use single_or_none so a real fork raises instead of being masked. --- simplyblock_core/controllers/backup/controller.py | 4 +++- .../controllers/backup/group_backup.py | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py index 37383559b7..a1b6efdb0f 100644 --- a/simplyblock_core/controllers/backup/controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -30,6 +30,7 @@ from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.utils.helpers import single_or_none logger = logging.getLogger() @@ -1160,7 +1161,8 @@ def delete_backup(backup_id): dependents = [b for b in db_controller.get_backups_by_lvol_id(backup.lvol_id) if b.prev_backup_id == backup.uuid] if dependents: - successor = next((b for b in dependents if b.status == Backup.STATUS_COMPLETED), None) + successor = single_or_none( + dependent for dependent in dependents if dependent.status == Backup.STATUS_COMPLETED) if successor is None: raise PreconditionError( f"backup {backup.uuid} is the predecessor of " diff --git a/simplyblock_core/controllers/backup/group_backup.py b/simplyblock_core/controllers/backup/group_backup.py index 6912d0dae6..0456549e2e 100644 --- a/simplyblock_core/controllers/backup/group_backup.py +++ b/simplyblock_core/controllers/backup/group_backup.py @@ -25,6 +25,7 @@ from simplyblock_core.controllers.backup import manifest as backup_manifest from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup, GroupBackup +from simplyblock_core.utils.helpers import single_or_none logger = utils.get_logger(__name__) @@ -666,7 +667,8 @@ def _retire_members(db, group_backup) -> bool: retired = False continue chain = db.get_backups_by_lvol_id(member.lvol_id) - successor = next((b for b in chain if b.prev_backup_id == member.uuid), None) + successor = single_or_none( + successor for successor in chain if successor.prev_backup_id == member.uuid) busy = (member.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS) or any(b.status == Backup.STATUS_MERGING for b in chain)) if successor is None and not busy: @@ -741,7 +743,16 @@ def retire_orphaned_member_backups(lvol_ids, db=None): db.get_group_backup_by_id(b.group_backup_id) except KeyError: orphans.append(b) - by_prev = {b.prev_backup_id: b for b in chain if b.prev_backup_id} + by_prev: dict = {} + for successor in chain: + if not successor.prev_backup_id: + continue + if successor.prev_backup_id in by_prev: + raise ValueError( + f"backup {successor.prev_backup_id} has multiple successors in " + f"volume {lvol_id}'s chain: {by_prev[successor.prev_backup_id].uuid} " + f"and {successor.uuid}") + by_prev[successor.prev_backup_id] = successor # Oldest first: the base of the chain is the one whose merge unblocks # the next, and a leaf is only a leaf once nothing is below it. for old in sorted(orphans, key=lambda b: (b.created_at, b.uuid)): From 317dc129c6b3a0a6b56560d8672840ba3f0699d1 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 18 Nov 2026 16:54:25 +0100 Subject: [PATCH 10/13] Fix claim_backup_chain_op allowing a conflicting re-claim of a target The own-targets exemption from the MERGING/DELETING block only checked that a backup's id was in target_ids, not that its current status already matched new_status. A backup claimed DELETING by one operation could therefore be silently re-claimed as MERGING by a different, concurrent one (e.g. trigger_merge racing delete_backup's earlier, non-transactional status read), overwriting the in-progress claim. Now a target only exempts itself when it is already in new_status; a target held in the other status still blocks, like any other sibling. --- simplyblock_core/db_controller.py | 24 ++++++++++--------- .../integration/backup/test_delete_backup.py | 17 +++++++++++++ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index 706d89255b..6119cd8434 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -1526,13 +1526,14 @@ def _claim_backup_chain_op_tx(self, tr, lvol_id, target_ids, new_status): target_set = set(target_ids) # A sibling already MERGING/DELETING blocks the claim UNLESS it is one - # of this call's own targets: re-claiming backups already in the state - # this call is about to set is what makes resuming an interrupted - # delete (the caller is retrying, not racing itself) safe, while a - # DIFFERENT concurrent operation's claim still refuses. + # of this call's own targets AND already sitting in new_status: that + # is what makes resuming an interrupted delete/merge (the caller is + # retrying, not racing itself) safe. A target held in the OTHER + # status -- claimed by a genuinely different operation -- still + # blocks, same as any other sibling. blocking = next((b for b in siblings.values() if b.status in (Backup.STATUS_MERGING, Backup.STATUS_DELETING) - and b.uuid not in target_set), None) + and not (b.uuid in target_set and b.status == new_status)), None) if blocking is not None: return False, f"backup {blocking.uuid} of this chain is already {blocking.status}" @@ -1568,12 +1569,13 @@ def claim_backup_chain_op(self, lvol_id, target_ids, new_status): by their bare ``uuid``, not ``get_id()``) for a destructive operation (merge or delete). - Refuses, as ``(False, reason)``, if any OTHER backup of the chain -- - one not in ``target_ids`` -- is already ``MERGING``/``DELETING``, or if - any backup in the chain (target or not) is named by an active - restore's own chain (``JobSchedule.restore_chain_s3_id``). On success, - every id in ``target_ids`` is flipped to ``new_status`` and - ``(True, None)`` is returned. + Refuses, as ``(False, reason)``, if any backup of the chain is already + ``MERGING``/``DELETING`` under a status other than ``new_status`` -- + whether or not it is one of ``target_ids`` -- or if any backup in the + chain (target or not) is named by an active restore's own chain + (``JobSchedule.restore_chain_s3_id``). On success, every id in + ``target_ids`` is flipped to ``new_status`` and ``(True, None)`` is + returned. Idempotent by design, not exclusive between identical claims: a target already in ``new_status`` -- this call resuming one that claimed it diff --git a/tests/integration/backup/test_delete_backup.py b/tests/integration/backup/test_delete_backup.py index 78a94946ac..bcb03c9b24 100644 --- a/tests/integration/backup/test_delete_backup.py +++ b/tests/integration/backup/test_delete_backup.py @@ -266,6 +266,23 @@ def test_a_chain_already_being_merged_refuses_a_delete_of_another_backup(db, buc assert _status(db, other_leaf) == Backup.STATUS_COMPLETED +def test_a_backup_claimed_for_delete_refuses_a_concurrent_merge_claim(db, bucket): + """claim_backup_chain_op's own-targets exemption must only cover a target + already sitting in new_status (resuming the identical claim) -- not a + target held in the OTHER status by a genuinely different operation.""" + base = _backup(db, bucket, "base") + _backup(db, bucket, "next", prev="base") + + ok, reason = db.claim_backup_chain_op(LVOL, [base.uuid], Backup.STATUS_DELETING) + assert ok, reason + + ok, reason = db.claim_backup_chain_op(LVOL, [base.uuid], Backup.STATUS_MERGING) + + assert not ok + assert base.uuid in reason + assert _status(db, base) == Backup.STATUS_DELETING + + def test_a_chain_read_by_an_active_restore_refuses_a_delete(db, bucket): leaf = _backup(db, bucket, "leaf") _restore_task(db, [leaf.s3_id]) From e21767a649c1c1aaff9646c53e7aa3b60e270941 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 18 Nov 2026 17:18:33 +0100 Subject: [PATCH 11/13] Fix fork detection in delete_backup to cover non-completed dependents single_or_none was applied only to the COMPLETED-filtered subset of dependents, so a fork with one COMPLETED sibling and one PENDING/ IN_PROGRESS sibling went undetected: the filter hid the non-completed one from the uniqueness check, and the completed one was merged into immediately. Check uniqueness over every dependent first, then the single survivor's status, so any fork is caught regardless of which sibling's status it involves. --- simplyblock_core/controllers/backup/controller.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py index a1b6efdb0f..c9061bec29 100644 --- a/simplyblock_core/controllers/backup/controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -1161,13 +1161,11 @@ def delete_backup(backup_id): dependents = [b for b in db_controller.get_backups_by_lvol_id(backup.lvol_id) if b.prev_backup_id == backup.uuid] if dependents: - successor = single_or_none( - dependent for dependent in dependents if dependent.status == Backup.STATUS_COMPLETED) - if successor is None: + successor = single_or_none(dependents) + if successor.status != Backup.STATUS_COMPLETED: raise PreconditionError( - f"backup {backup.uuid} is the predecessor of " - f"{', '.join(d.uuid[:8] for d in dependents)}, none of which is " - "completed yet; it cannot be merged or deleted") + f"backup {backup.uuid} is the predecessor of {successor.uuid}, " + "which is not completed yet; it cannot be merged or deleted") ok, reason = trigger_merge(successor, backup) if not ok: raise PreconditionError(f"Cannot delete backup {backup.uuid}: {reason}") From 50eaf1bfb29c4782caf43dc87554227631c2e299 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Wed, 18 Nov 2026 17:22:39 +0100 Subject: [PATCH 12/13] Add regression tests for the chain-fork and claim-race fixes Covers the three fixes that lacked a dedicated test: - delete_backup raising on a predecessor with multiple dependents of any status (controller.py), not just multiple COMPLETED ones. - _retire_members raising on a forked successor lookup (group_backup.py). - retire_orphaned_member_backups raising on a forked by_prev index instead of silently keeping whichever backup the construction saw last. claim_backup_chain_op's fix already had a regression test added earlier (test_a_backup_claimed_for_delete_refuses_a_concurrent_merge_claim). --- .../integration/backup/test_delete_backup.py | 33 +++++++++++++++++++ .../backup/test_group_backup_delete.py | 32 ++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/tests/integration/backup/test_delete_backup.py b/tests/integration/backup/test_delete_backup.py index bcb03c9b24..1b02760ca8 100644 --- a/tests/integration/backup/test_delete_backup.py +++ b/tests/integration/backup/test_delete_backup.py @@ -22,6 +22,7 @@ from botocore.exceptions import ClientError from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import group_backup from simplyblock_core.controllers.backup import manifest as backup_manifest from simplyblock_core.controllers.backup.manifest import ( BackupManifest, @@ -228,6 +229,23 @@ def test_a_backup_with_an_incomplete_successor_is_refused(db, bucket): assert _status(db, base) == Backup.STATUS_COMPLETED +def test_a_forked_predecessor_raises_instead_of_merging_into_one_arbitrarily(db, bucket): + """Two backups naming the same predecessor is a chain fork that should + never occur (snapshot chains are linear and backup creation locks the + whole ancestry) -- if one gets into the chain anyway (e.g. a corrupted + import), deleting the predecessor must fail loudly, not silently merge it + into whichever fork happens to be completed and ignore the other, even + one still being uploaded.""" + base = _backup(db, bucket, "base") + _backup(db, bucket, "completed_fork", prev="base") + _backup(db, bucket, "uploading_fork", prev="base", status=Backup.STATUS_IN_PROGRESS) + + with pytest.raises(ValueError, match="Multiple values present"): + backup_controller.delete_backup(base.uuid) + + assert _status(db, base) == Backup.STATUS_COMPLETED + + def test_a_member_of_a_group_backup_that_exists_is_refused(db, bucket): gb = _group_backup(db) member = _backup(db, bucket, "member", group_backup_id=gb.get_id()) @@ -246,6 +264,21 @@ def test_a_member_whose_group_backup_is_gone_is_queued_for_deletion(db, bucket): assert _status(db, orphan) == Backup.STATUS_DELETING +def test_retire_orphaned_member_backups_raises_on_a_forked_chain(db, bucket): + """retire_orphaned_member_backups indexes a chain by prev_backup_id to + find each orphan's successor; two backups sharing one predecessor must + raise instead of silently keeping whichever the index construction saw + last and losing the other.""" + orphan = _backup(db, bucket, "orphan", group_backup_id=f"{CLUSTER}/{_uuid('gone')}") + _backup(db, bucket, "next_a", prev="orphan") + _backup(db, bucket, "next_b", prev="orphan") + + with pytest.raises(ValueError, match="multiple successors"): + group_backup.retire_orphaned_member_backups([LVOL], db=db) + + assert _status(db, orphan) == Backup.STATUS_COMPLETED + + def test_an_unknown_backup_raises(db): with pytest.raises(KeyError): backup_controller.delete_backup(_uuid("nope")) diff --git a/tests/integration/backup/test_group_backup_delete.py b/tests/integration/backup/test_group_backup_delete.py index 4a9abcca98..e3573086fd 100644 --- a/tests/integration/backup/test_group_backup_delete.py +++ b/tests/integration/backup/test_group_backup_delete.py @@ -147,6 +147,23 @@ def _two_generations(db, bucket): return gb, base, leaf, successor +def _rogue_backup(db, uuid, lvol_id, prev_backup_id): + """A backup outside any group backup that chains against *prev_backup_id* + -- for building a chain fork: two backups naming the same predecessor.""" + b = Backup() + b.uuid = uuid + b.cluster_id = CLUSTER + b.lvol_id = lvol_id + b.node_id = NODE + b.prev_backup_id = prev_backup_id + b.size = 4096 + b.created_at = 999 + b.status = Backup.STATUS_COMPLETED + b.location = _config().location().model_dump(mode="json") + b.write_to_db(db.kv_store) + return b + + def _status(db, backup): try: return db.get_backup_by_id(backup.get_id()).status @@ -253,6 +270,21 @@ def test_the_next_generation_is_left_whole(self, db, bucket): assert db.get_group_backup_by_id(f"{CLUSTER}/{GEN_8}").status == GroupBackup.STATUS_COMPLETED assert bucket[GEN_8].state == GroupManifestState.COMMITTED + def test_a_forked_base_raises_instead_of_picking_a_successor_arbitrarily(self, db, bucket): + """Two backups naming the same predecessor is a chain fork that should + never occur (snapshot chains are linear and backup creation locks the + whole ancestry) -- if one gets into the chain anyway (e.g. a corrupted + import), retiring the predecessor must fail loudly, not silently merge + it into whichever fork next() happens to see first.""" + gb, base, leaf, successor = _two_generations(db, bucket) + _rogue_backup(db, "0ae09999-0000-4000-8000-000000000099", base.lvol_id, base.uuid) + group_backup.delete_group_backup(gb.get_id()) + + with pytest.raises(ValueError, match="Multiple values present"): + group_backup.advance_group_backup_delete(gb.get_id()) + + assert _status(db, base) == Backup.STATUS_COMPLETED + def test_a_failed_group_backup_has_no_manifest_to_mark(self, db, bucket): member = _member(db, GEN_7, 7, 1, status=Backup.STATUS_FAILED) gb = _group_backup(db, bucket, GEN_7, 7, [member], status=GroupBackup.STATUS_FAILED) From 665bd31406a0d1a0f5c9aeb7d691a7cca01ea6db Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 20 Nov 2026 17:19:32 +0100 Subject: [PATCH 13/13] Replace ad hoc backup-chain fork checks with SuccessorIndex delete_backup, _retire_members and retire_orphaned_member_backups each hand-rolled their own prev_backup_id lookup to find a backup's successor, with inconsistent fork detection (single_or_none's ValueError in two places, a manual dict in the third). Add SuccessorIndex to chain.py -- the forward-lookup counterpart to BackupChain's backward walk -- and migrate all three sites to it, so a chain fork is now always reported as PreconditionError from one place. Also points simplyblock_core/AGENTS.md at chain.py as the canonical home for backup-chain invariants, since the original backup-deletion implementation missed BackupChain entirely due to the module's docstring being framed only around restore. --- simplyblock_core/AGENTS.md | 2 +- simplyblock_core/controllers/backup/chain.py | 47 +++++++++++++++++++ .../controllers/backup/controller.py | 14 +++--- .../controllers/backup/group_backup.py | 17 ++----- .../integration/backup/test_delete_backup.py | 4 +- .../backup/test_group_backup_delete.py | 2 +- tests/unit/backup/test_chain.py | 42 ++++++++++++++++- 7 files changed, 103 insertions(+), 25 deletions(-) diff --git a/simplyblock_core/AGENTS.md b/simplyblock_core/AGENTS.md index e4438ee96d..ec1895103d 100644 --- a/simplyblock_core/AGENTS.md +++ b/simplyblock_core/AGENTS.md @@ -4,7 +4,7 @@ Core business logic, data models, and background services for the Simplyblock co ## Package Structure -- `controllers/` — Business logic per resource domain (lvol, snapshot, backup, device, migration, pool, health, tasks, qos). Each `*_events.py` defines event types for its domain. +- `controllers/` — Business logic per resource domain (lvol, snapshot, backup, device, migration, pool, health, tasks, qos). Each `*_events.py` defines event types for its domain. For the backup domain, `controllers/backup/chain.py` is the canonical home for backup-chain invariants — coherence (`BackupChain`, restore-side) and fork detection (`SuccessorIndex`, delete/retire-side). Check there before hand-rolling a chain walk or a `prev_backup_id` lookup at a new call site. - `models/` — Data models inheriting from `BaseModel` (see below), plus the two stdlib-only modules that define the keyspaces those records live in: `watches.py` (the version index watchers wake on) and `indices.py` (the declared secondary indices). Both are leaves — the diff --git a/simplyblock_core/controllers/backup/chain.py b/simplyblock_core/controllers/backup/chain.py index cfa0f4f3cf..39d70a4ff9 100644 --- a/simplyblock_core/controllers/backup/chain.py +++ b/simplyblock_core/controllers/backup/chain.py @@ -324,3 +324,50 @@ def s3_ids_newest_first(self) -> list[int]: data has to win and older backups fill whatever gaps remain. """ return [link.s3_id for link in reversed(self.links)] + + +@dataclass(frozen=True) +class SuccessorIndex: + """Which backup, if any, is a delta against each backup of one lvol. + + The complement of :class:`BackupChain`: that walks backward from a trusted + head; this answers the forward question delete and retire need -- "does + anything chain against this backup" -- over a whole lvol's backups at once, + with no head and no restorability or coherence requirement, since nothing + here is being restored. + + A fork -- two backups naming the same ``prev_backup_id`` -- can only reach + storage through backup import, which checks each manifest's own ancestry + but never checks two manifests, or a manifest against an existing record, + against each other. :meth:`of_lvol` is where that defect becomes visible: + it refuses to guess which of the two is "the" successor. + """ + + _by_prev: Mapping[str, Backup] + + @classmethod + def of_lvol(cls, backups: Iterable[Backup]) -> "SuccessorIndex": + """The successor of every backup among *backups*, which should be + every ``Backup`` of one lvol -- a successor outside that population + would not be found. + + Raises: + PreconditionError: two backups name the same ``prev_backup_id``, + i.e. the chain has forked. + """ + by_prev: dict[str, Backup] = {} + for backup in backups: + if not backup.prev_backup_id: + continue + if (existing := by_prev.get(backup.prev_backup_id)) is not None: + raise PreconditionError( + f"backup {backup.prev_backup_id} has multiple successors: " + f"{existing.uuid} and {backup.uuid}") + by_prev[backup.prev_backup_id] = backup + + return cls(by_prev) + + def successor_of(self, backup_id: str) -> Backup | None: + """The backup that is a delta against *backup_id*, or ``None`` if + nothing in this index chains against it.""" + return self._by_prev.get(backup_id) diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py index c9061bec29..b894e75e1d 100644 --- a/simplyblock_core/controllers/backup/controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -8,7 +8,11 @@ from simplyblock_core.controllers import backup_events, tasks_controller from simplyblock_core.controllers.backup import manifest as backup_manifest -from simplyblock_core.controllers.backup.chain import BackupChain, location_of +from simplyblock_core.controllers.backup.chain import ( + BackupChain, + SuccessorIndex, + location_of, +) from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.kms import ( @@ -30,7 +34,6 @@ from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.storage_node import StorageNode -from simplyblock_core.utils.helpers import single_or_none logger = logging.getLogger() @@ -1158,10 +1161,9 @@ def delete_backup(backup_id): backup.remove(db_controller.kv_store) return - dependents = [b for b in db_controller.get_backups_by_lvol_id(backup.lvol_id) - if b.prev_backup_id == backup.uuid] - if dependents: - successor = single_or_none(dependents) + successor = SuccessorIndex.of_lvol( + db_controller.get_backups_by_lvol_id(backup.lvol_id)).successor_of(backup.uuid) + if successor is not None: if successor.status != Backup.STATUS_COMPLETED: raise PreconditionError( f"backup {backup.uuid} is the predecessor of {successor.uuid}, " diff --git a/simplyblock_core/controllers/backup/group_backup.py b/simplyblock_core/controllers/backup/group_backup.py index 0456549e2e..bcbfc7278e 100644 --- a/simplyblock_core/controllers/backup/group_backup.py +++ b/simplyblock_core/controllers/backup/group_backup.py @@ -25,7 +25,6 @@ from simplyblock_core.controllers.backup import manifest as backup_manifest from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup, GroupBackup -from simplyblock_core.utils.helpers import single_or_none logger = utils.get_logger(__name__) @@ -667,8 +666,7 @@ def _retire_members(db, group_backup) -> bool: retired = False continue chain = db.get_backups_by_lvol_id(member.lvol_id) - successor = single_or_none( - successor for successor in chain if successor.prev_backup_id == member.uuid) + successor = backup_chain.SuccessorIndex.of_lvol(chain).successor_of(member.uuid) busy = (member.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS) or any(b.status == Backup.STATUS_MERGING for b in chain)) if successor is None and not busy: @@ -743,20 +741,11 @@ def retire_orphaned_member_backups(lvol_ids, db=None): db.get_group_backup_by_id(b.group_backup_id) except KeyError: orphans.append(b) - by_prev: dict = {} - for successor in chain: - if not successor.prev_backup_id: - continue - if successor.prev_backup_id in by_prev: - raise ValueError( - f"backup {successor.prev_backup_id} has multiple successors in " - f"volume {lvol_id}'s chain: {by_prev[successor.prev_backup_id].uuid} " - f"and {successor.uuid}") - by_prev[successor.prev_backup_id] = successor + successors = backup_chain.SuccessorIndex.of_lvol(chain) # Oldest first: the base of the chain is the one whose merge unblocks # the next, and a leaf is only a leaf once nothing is below it. for old in sorted(orphans, key=lambda b: (b.created_at, b.uuid)): - successor = by_prev.get(old.uuid) + successor = successors.successor_of(old.uuid) if successor is None: try: backup_controller.delete_backup(old.uuid) diff --git a/tests/integration/backup/test_delete_backup.py b/tests/integration/backup/test_delete_backup.py index 1b02760ca8..9b3dd75d5f 100644 --- a/tests/integration/backup/test_delete_backup.py +++ b/tests/integration/backup/test_delete_backup.py @@ -240,7 +240,7 @@ def test_a_forked_predecessor_raises_instead_of_merging_into_one_arbitrarily(db, _backup(db, bucket, "completed_fork", prev="base") _backup(db, bucket, "uploading_fork", prev="base", status=Backup.STATUS_IN_PROGRESS) - with pytest.raises(ValueError, match="Multiple values present"): + with pytest.raises(PreconditionError, match="multiple successors"): backup_controller.delete_backup(base.uuid) assert _status(db, base) == Backup.STATUS_COMPLETED @@ -273,7 +273,7 @@ def test_retire_orphaned_member_backups_raises_on_a_forked_chain(db, bucket): _backup(db, bucket, "next_a", prev="orphan") _backup(db, bucket, "next_b", prev="orphan") - with pytest.raises(ValueError, match="multiple successors"): + with pytest.raises(PreconditionError, match="multiple successors"): group_backup.retire_orphaned_member_backups([LVOL], db=db) assert _status(db, orphan) == Backup.STATUS_COMPLETED diff --git a/tests/integration/backup/test_group_backup_delete.py b/tests/integration/backup/test_group_backup_delete.py index e3573086fd..c37a3007a6 100644 --- a/tests/integration/backup/test_group_backup_delete.py +++ b/tests/integration/backup/test_group_backup_delete.py @@ -280,7 +280,7 @@ def test_a_forked_base_raises_instead_of_picking_a_successor_arbitrarily(self, d _rogue_backup(db, "0ae09999-0000-4000-8000-000000000099", base.lvol_id, base.uuid) group_backup.delete_group_backup(gb.get_id()) - with pytest.raises(ValueError, match="Multiple values present"): + with pytest.raises(PreconditionError, match="multiple successors"): group_backup.advance_group_backup_delete(gb.get_id()) assert _status(db, base) == Backup.STATUS_COMPLETED diff --git a/tests/unit/backup/test_chain.py b/tests/unit/backup/test_chain.py index 9f9fbdbecd..ac19d421a4 100644 --- a/tests/unit/backup/test_chain.py +++ b/tests/unit/backup/test_chain.py @@ -14,7 +14,11 @@ import pytest from simplyblock_core import constants -from simplyblock_core.controllers.backup.chain import BackupChain, location_of +from simplyblock_core.controllers.backup.chain import ( + BackupChain, + SuccessorIndex, + location_of, +) from simplyblock_core.controllers.backup.manifest import ( BackupManifest, FDBKeyDescriptor, @@ -273,6 +277,42 @@ def test_the_wording_names_what_was_refused(self): what="This snapshot chain") +class TestSuccessorIndex: + + def test_a_backup_with_no_dependent_has_no_successor(self): + index = SuccessorIndex.of_lvol(_line_of_records(2)) + assert index.successor_of(str(_id("b-1"))) is None + + def test_finds_the_backup_naming_it_as_prev(self): + line = _line_of_records(3) + index = SuccessorIndex.of_lvol(line) + successor_of_0 = index.successor_of(str(_id("b-0"))) + successor_of_1 = index.successor_of(str(_id("b-1"))) + assert successor_of_0 is not None and successor_of_0.uuid == str(_id("b-1")) + assert successor_of_1 is not None and successor_of_1.uuid == str(_id("b-2")) + + def test_order_of_the_population_does_not_matter(self): + line = _line_of_records(3) + index = SuccessorIndex.of_lvol(list(reversed(line))) + successor = index.successor_of(str(_id("b-0"))) + assert successor is not None and successor.uuid == str(_id("b-1")) + + def test_a_backup_naming_no_predecessor_is_ignored(self): + """A full backup's empty prev_backup_id must not collide with another + full backup's, as if both were "the successor of nothing".""" + a = _record("full-a") + b = _record("full-b") + index = SuccessorIndex.of_lvol([a, b]) + assert index.successor_of("") is None + + def test_two_backups_naming_the_same_predecessor_is_a_fork(self): + base = _record("base") + fork_a = _record("fork-a", prev="base") + fork_b = _record("fork-b", prev="base") + with pytest.raises(PreconditionError, match="multiple successors"): + SuccessorIndex.of_lvol([base, fork_a, fork_b]) + + class TestS3Ids: def test_newest_first(self):