diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 2b907415aa..c5c25ee353 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1763,6 +1763,25 @@ commands: dest: repair type: bool action: store_true + - name: check-backups + help: "Verifies every backup's FDB record against its S3 manifest." + usage: > + Checks, in both directions, that every COMPLETED or DELETING backup has a manifest whose state + and s3_id/prev_backup_id agree with the record, and that every ACTIVE manifest in the bucket + names a record that still exists. Read-only unless `--repair` is given, which rewrites a + missing or disagreeing manifest from the record (today's source of truth). An orphaned + manifest - one naming no known record - is never auto-repaired: it may be a disaster-recovery + backup nothing has imported yet. Deployment-wide unless --cluster-id is given. + arguments: + - name: "--cluster-id" + help: "Limit the check to one cluster. Omit to check every cluster." + dest: cluster_id + type: str + - name: "--repair" + help: "Rewrite manifests that are missing or disagree with their record, instead of only reporting them." + dest: repair + type: bool + action: store_true - name: graceful-shutdown help: "Initiates a graceful shutdown of a cluster's storage nodes." arguments: @@ -3364,6 +3383,13 @@ commands: help: "The logical volume id." dest: lvol_id type: str + - name: delete-backup + help: "Delete one backup. A backup nothing chains against is deleted; one something chains against is merged into that successor instead." + arguments: + - name: "backup_id" + help: "The backup id." + dest: backup_id + type: str - name: restore help: "Restore a backup to a new logical volume." arguments: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 0d21c5fcb1..1f1cb48125 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -399,6 +399,7 @@ def init_cluster(self): self.init_cluster__build_indices(subparser) self.init_cluster__index_state(subparser) self.init_cluster__check_indices(subparser) + self.init_cluster__check_backups(subparser) self.init_cluster__graceful_shutdown(subparser) self.init_cluster__restart(subparser) self.init_cluster__graceful_startup(subparser) @@ -645,6 +646,11 @@ def init_cluster__check_indices(self, subparser): subcommand = self.add_sub_command(subparser, 'check-indices', 'Verifies the database\'s secondary indices.') subcommand.add_argument('--repair', help='Write back the entries that can be derived from the records, and clear orphaned ones, instead of only reporting them.', dest='repair', action='store_true') + def init_cluster__check_backups(self, subparser): + subcommand = self.add_sub_command(subparser, 'check-backups', 'Verifies every backup\'s FDB record against its S3 manifest.') + subcommand.add_argument('--cluster-id', help='Limit the check to one cluster. Omit to check every cluster.', type=str, dest='cluster_id') + subcommand.add_argument('--repair', help='Rewrite manifests that are missing or disagree with their record, instead of only reporting them.', dest='repair', action='store_true') + def init_cluster__graceful_shutdown(self, subparser): subcommand = self.add_sub_command(subparser, 'graceful-shutdown', 'Initiates a graceful shutdown of a cluster\'s storage nodes.') subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list @@ -1310,6 +1316,7 @@ def init_backup(self): subparser = self.add_command('backup', 'Backup Commands') self.init_backup__list(subparser) self.init_backup__delete(subparser) + self.init_backup__delete_backup(subparser) self.init_backup__restore(subparser) self.init_backup__export(subparser) self.init_backup__discover(subparser) @@ -1330,6 +1337,10 @@ def init_backup__delete(self, subparser): subcommand = self.add_sub_command(subparser, 'delete', 'Delete all backups for a logical volume.') subcommand.add_argument('lvol_id', help='The logical volume id.', type=str) + def init_backup__delete_backup(self, subparser): + subcommand = self.add_sub_command(subparser, 'delete-backup', 'Delete one backup. A backup nothing chains against is deleted; one something chains against is merged into that successor instead.') + subcommand.add_argument('backup_id', help='The backup id.', type=str) + def init_backup__restore(self, subparser): subcommand = self.add_sub_command(subparser, 'restore', 'Restore a backup to a new logical volume.') subcommand.add_argument('backup_id', help='The volume backup id.', type=str) @@ -1760,6 +1771,8 @@ def run(self): ret = self.cluster__index_state(sub_command, args) elif sub_command in ['check-indices']: ret = self.cluster__check_indices(sub_command, args) + elif sub_command in ['check-backups']: + ret = self.cluster__check_backups(sub_command, args) elif sub_command in ['graceful-shutdown']: ret = self.cluster__graceful_shutdown(sub_command, args) elif sub_command in ['restart']: @@ -2017,6 +2030,8 @@ def run(self): ret = self.backup__list(sub_command, args) elif sub_command in ['delete']: ret = self.backup__delete(sub_command, args) + elif sub_command in ['delete-backup']: + ret = self.backup__delete_backup(sub_command, args) elif sub_command in ['restore']: ret = self.backup__restore(sub_command, args) elif sub_command in ['export']: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 3a9b470dc6..04c41790f1 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -767,6 +767,10 @@ def cluster__index_state(self, sub_command, args): def cluster__check_indices(self, sub_command, args): return cluster_ops.check_indices(repair=args.repair) + def cluster__check_backups(self, sub_command, args): + return cluster_ops.check_backups( + cluster_id=getattr(args, 'cluster_id', None), repair=args.repair) + def cluster__graceful_shutdown(self, sub_command, args): cluster_ops.cluster_grace_shutdown(args.cluster_id) return True @@ -1773,8 +1777,16 @@ def backup__delete(self, sub_command, args): if error: print(f"Error: {error}") return False - print("Backups deleted") - return True + return f"Backups of volume {args.lvol_id} are being deleted" + + def backup__delete_backup(self, sub_command, args): + try: + backup_controller.delete_backup(args.backup_id) + except KeyError: + return f"Delete failed: backup {args.backup_id} not found" + except PreconditionError as e: + return f"Delete failed: {e}" + return f"Backup {args.backup_id} is being deleted" def backup__restore(self, sub_command, args): try: diff --git a/simplyblock_core/AGENTS.md b/simplyblock_core/AGENTS.md index e4438ee96d..ec1895103d 100644 --- a/simplyblock_core/AGENTS.md +++ b/simplyblock_core/AGENTS.md @@ -4,7 +4,7 @@ Core business logic, data models, and background services for the Simplyblock co ## Package Structure -- `controllers/` — Business logic per resource domain (lvol, snapshot, backup, device, migration, pool, health, tasks, qos). Each `*_events.py` defines event types for its domain. +- `controllers/` — Business logic per resource domain (lvol, snapshot, backup, device, migration, pool, health, tasks, qos). Each `*_events.py` defines event types for its domain. For the backup domain, `controllers/backup/chain.py` is the canonical home for backup-chain invariants — coherence (`BackupChain`, restore-side) and fork detection (`SuccessorIndex`, delete/retire-side). Check there before hand-rolling a chain walk or a `prev_backup_id` lookup at a new call site. - `models/` — Data models inheriting from `BaseModel` (see below), plus the two stdlib-only modules that define the keyspaces those records live in: `watches.py` (the version index watchers wake on) and `indices.py` (the declared secondary indices). Both are leaves — the diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index f154629c80..f2a4ce0b60 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -52,6 +52,7 @@ tasks_controller, tcp_ports_events, ) +from simplyblock_core.controllers.backup import controller as backup_controller from simplyblock_core.controllers.backup import device as backup_device from simplyblock_core.controllers.cluster_expansion.planner import ( RemoteTripletPlacementError, @@ -4323,6 +4324,16 @@ def check_indices(repair=False) -> bool: return findings['unresolved'] == 0 +def check_backups(cluster_id=None, repair=False) -> bool: + """Verify every backup's FDB record agrees with its S3 manifest, in both + directions, and report the drift. The backup analog of + :func:`check_indices`. + + Deployment-wide when ``cluster_id`` is omitted. + """ + return backup_controller.check_backups(cluster_id=cluster_id, repair=repair) + + def cluster_grace_startup(cl_id, clear_data=False, spdk_image=None) -> None: get_cluster = db_controller.get_cluster_by_id(cl_id) # ensure exists diff --git a/simplyblock_core/controllers/backup/chain.py b/simplyblock_core/controllers/backup/chain.py index cfa0f4f3cf..39d70a4ff9 100644 --- a/simplyblock_core/controllers/backup/chain.py +++ b/simplyblock_core/controllers/backup/chain.py @@ -324,3 +324,50 @@ def s3_ids_newest_first(self) -> list[int]: data has to win and older backups fill whatever gaps remain. """ return [link.s3_id for link in reversed(self.links)] + + +@dataclass(frozen=True) +class SuccessorIndex: + """Which backup, if any, is a delta against each backup of one lvol. + + The complement of :class:`BackupChain`: that walks backward from a trusted + head; this answers the forward question delete and retire need -- "does + anything chain against this backup" -- over a whole lvol's backups at once, + with no head and no restorability or coherence requirement, since nothing + here is being restored. + + A fork -- two backups naming the same ``prev_backup_id`` -- can only reach + storage through backup import, which checks each manifest's own ancestry + but never checks two manifests, or a manifest against an existing record, + against each other. :meth:`of_lvol` is where that defect becomes visible: + it refuses to guess which of the two is "the" successor. + """ + + _by_prev: Mapping[str, Backup] + + @classmethod + def of_lvol(cls, backups: Iterable[Backup]) -> "SuccessorIndex": + """The successor of every backup among *backups*, which should be + every ``Backup`` of one lvol -- a successor outside that population + would not be found. + + Raises: + PreconditionError: two backups name the same ``prev_backup_id``, + i.e. the chain has forked. + """ + by_prev: dict[str, Backup] = {} + for backup in backups: + if not backup.prev_backup_id: + continue + if (existing := by_prev.get(backup.prev_backup_id)) is not None: + raise PreconditionError( + f"backup {backup.prev_backup_id} has multiple successors: " + f"{existing.uuid} and {backup.uuid}") + by_prev[backup.prev_backup_id] = backup + + return cls(by_prev) + + def successor_of(self, backup_id: str) -> Backup | None: + """The backup that is a delta against *backup_id*, or ``None`` if + nothing in this index chains against it.""" + return self._by_prev.get(backup_id) diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py index b0a21610db..b894e75e1d 100644 --- a/simplyblock_core/controllers/backup/controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -8,7 +8,11 @@ from simplyblock_core.controllers import backup_events, tasks_controller from simplyblock_core.controllers.backup import manifest as backup_manifest -from simplyblock_core.controllers.backup.chain import BackupChain, location_of +from simplyblock_core.controllers.backup.chain import ( + BackupChain, + SuccessorIndex, + location_of, +) from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.kms import ( @@ -30,7 +34,6 @@ from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.storage_node import StorageNode -from simplyblock_core.rpc_client import RPCException logger = logging.getLogger() @@ -378,6 +381,47 @@ def delete_manifest(backup: Backup) -> None: backup_manifest.delete(_config_for(backup), UUID(backup.uuid)) +def delete_backup_data(backup: Backup) -> None: + """Remove a backup's S3 data objects directly, with no data-plane RPC. + + There is no data-plane RPC for this and none is planned -- a bucket's data + objects can be removed the same way a manifest is, by the control plane + talking to S3 itself. This is only for a backup whose delete needs no + data-plane involvement (a tip delete, or a whole-chain delete): a backup + something still chains against is folded into its successor by + ``trigger_merge`` instead, which is where the data plane's own + ``bdev_lvol_s3_merge`` remains the only way to remove such a backup's + objects without breaking its descendant. + + Raises: + ManifestError: the objects could not be listed or removed. + PreconditionError: the backup's bucket is not the cluster's own. + """ + backup_manifest.delete_data_objects(_config_for(backup), backup.s3_id) + + +def mark_manifest_deleting(backup: Backup) -> None: + """Mark a backup's manifest DELETING, before any of its S3 objects are + touched. See ``manifest.mark_deleting``. + + Raises: + ManifestError: the manifest could not be read or rewritten. + PreconditionError: the backup's bucket is not the cluster's own. + """ + backup_manifest.mark_deleting(_config_for(backup), UUID(backup.uuid)) + + +def revert_manifest_deleting(backup: Backup) -> None: + """Revert a backup's manifest to ACTIVE: a delete that did not finish. + See ``manifest.mark_active``. + + Raises: + ManifestError: the manifest could not be read or rewritten. + PreconditionError: the backup's bucket is not the cluster's own. + """ + backup_manifest.mark_active(_config_for(backup), UUID(backup.uuid)) + + def _blob_parent_id(snapshot: SnapShot) -> str | None: """The snapshot this one's blob is a delta against, or ``None`` at a root. @@ -927,15 +971,16 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, bdev_name = f"{lvol.lvs_name}/{lvol.lvol_bdev}" if not tasks_controller.add_backup_restore_task( - pool.cluster_id, lvol.node_id, backup_id, bdev_name, + backup.lvol_id, pool.cluster_id, lvol.node_id, backup_id, bdev_name, chain.s3_ids_newest_first(), lvol_id=lvol_id, s3_config=s3_config.model_dump(exclude_none=True) if s3_config is not None else None): - raise RuntimeError("Failed to create restore task") + raise PreconditionError( + "Failed to create restore task: the source chain is being merged or deleted") return lvol_id -def _cleanup_backup_kms_keys(backups): +def cleanup_backup_kms_keys(backups): encrypted = [b for b in backups if b.encrypted] if not encrypted: return @@ -952,15 +997,75 @@ def _cleanup_backup_kms_keys(backups): logger.exception("Failed to clean up backup KMS keys") +def _revert_backup_claim(backup_id: str) -> None: + """Undo a claim made by :func:`_start_backup_delete`/``trigger_merge`` that + did not get as far as anything irreversible: put the backup back to + COMPLETED. + + A no-op if the backup is gone (improbable here -- nothing between the + claim and this has removed it) or has moved past DELETING on its own. + """ + try: + fresh = db_controller.get_backup_by_id(backup_id) + except KeyError: + return + if fresh.status != Backup.STATUS_DELETING: + return + db_controller.atomic_update(fresh, lambda b: setattr(b, "status", Backup.STATUS_COMPLETED)) + + +def _start_backup_delete(backups: list[Backup]): + """Claim every backup in ``backups`` -- all COMPLETED, all belonging to one + volume's chain -- for deletion, mark their manifests DELETING, and queue + the task that removes their S3 objects. + + All-or-nothing: the chain claim (``DBController.claim_backup_chain_op``) + is one transaction already, and if marking any manifest afterwards fails, + every claim just taken is reverted back to COMPLETED before this returns + -- so a partial failure here never leaves a backup DELETING with an + ACTIVE manifest, or a manifest DELETING with no FDB record to match it + (the manifest write only happens after the claim, never before). + + Returns ``(True, None)`` once the delete task is queued (or an equivalent + one already was -- ``add_backup_delete_task`` dedupes), or + ``(False, reason)``. + """ + if not backups: + return True, None + lvol_id = backups[0].lvol_id + ids = [b.uuid for b in backups] + + ok, reason = db_controller.claim_backup_chain_op(lvol_id, ids, Backup.STATUS_DELETING) + if not ok: + return False, reason + + try: + for b in backups: + mark_manifest_deleting(b) + except (backup_manifest.ManifestError, PreconditionError, ValueError, KeyError) as e: + # KeyError: _config_for's own cluster lookup, inside + # mark_manifest_deleting. Everything here ran after the claim already + # committed, so every target is reverted -- there is nothing partial + # to distinguish. + for backup_id in ids: + _revert_backup_claim(backup_id) + return False, str(e) + + tasks_controller.add_backup_delete_task(backups[0].cluster_id, backups[0].node_id, ids) + return True, None + + def delete_backups(lvol_id): - """Delete all backups for a given lvol. + """Delete every backup of a volume. - Removes the database records, not the objects: bdev_lvol_s3_delete does not - exist on the data plane, so the S3 data outlives this call. The manifests - are deliberately left in place too -- they are the only thing that can still - identify those objects, and deleting them would turn a reclaimable orphan - set into anonymous bucket weight. `backup discover` therefore keeps showing - them, which is the honest answer about what the bucket contains. + All of them go directly to deletion, never a merge: nothing outside this + call survives to chain against any of them, so there is nothing to fold + one into. Refuses outright if any backup of the volume is still being + created (PENDING/IN_PROGRESS) -- deleting mid-upload would remove the very + backup a running upload's own ``prev_backup_id`` depends on -- or if any + is a live member of a group backup, or already MERGING (someone else's + merge of a backup this call would otherwise also claim). A backup already + DELETING is tolerated: this call resuming one that did not finish. Returns (success, error_message). """ @@ -968,65 +1073,66 @@ def delete_backups(lvol_id): if not backups: return False, f"No backups found for lvol {lvol_id}" - _cleanup_backup_kms_keys(backups) - - # Find node to run delete RPC on - completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] - if not completed: - # Just remove from DB - for b in backups: - b.remove(db_controller.kv_store) - return True, None + in_flight = [b for b in backups + if b.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS)] + if in_flight: + return False, (f"backup(s) {', '.join(b.uuid[:8] for b in in_flight)} of volume " + f"{lvol_id} are still being created") - node_id = completed[0].node_id - try: - snode = db_controller.get_storage_node_by_id(node_id) - except KeyError: - # Node gone, just clean up DB - for b in backups: - b.remove(db_controller.kv_store) - return True, None + merging = [b for b in backups if b.status == Backup.STATUS_MERGING] + if merging: + return False, (f"backup(s) {', '.join(b.uuid[:8] for b in merging)} of volume " + f"{lvol_id} are already being merged") - # Call S3 delete RPC (dummy for now) - if snode.status == StorageNode.STATUS_ONLINE: - rpc_client = snode.rpc_client() - s3_ids = [b.s3_id for b in completed] + for b in backups: + if not b.group_backup_id: + continue try: - rpc_client.bdev_lvol_s3_delete(s3_ids) - except Exception as e: - logger.error(f"Error deleting S3 backups: {e}") + owner = db_controller.get_group_backup_by_id(b.group_backup_id) + except KeyError: + continue # its group backup is gone: an orphaned member copy + if owner.status != GroupBackup.STATUS_DELETING: + return False, (f"backup {b.uuid} belongs to group backup " + f"{b.group_backup_id}, which still exists; delete the " + "group backup instead") - cluster_id = completed[0].cluster_id + completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] + if completed: + ok, reason = _start_backup_delete(completed) + if not ok: + return False, f"Cannot delete backups of volume {lvol_id}: {reason}" + + # A FAILED backup uploaded nothing restorable and has no manifest: remove + # its record directly. COMPLETED backups are left for the task just + # queued above to retire; a backup already DELETING is being retired by + # that task or an earlier one. for b in backups: - backup_events.backup_deleted(cluster_id, node_id, b) + if b.status != Backup.STATUS_FAILED: + continue + cleanup_backup_kms_keys([b]) + backup_events.backup_deleted(b.cluster_id, b.node_id, b) b.remove(db_controller.kv_store) return True, None def delete_backup(backup_id): - """Delete ONE backup: its record, its KMS keys and, best effort, its S3 - objects -- the single-record form of :func:`delete_backups`, for a backup - that nothing chains against any more, such as a deleted group backup's - member that is the last link of its volume's chain. + """Delete ONE backup: start deleting it if nothing chains against it, or + fold it into its completed successor if something does -- deleting a + backup with a successor and merging it into that successor are the same + request; see :func:`trigger_merge`. Raises: KeyError: no backup has this id. - PreconditionError: another backup names this one as its predecessor, - so deleting it would make every descendant unrestorable -- merge it - into its successor instead (``trigger_merge``), after which it is - MERGED, not deleted, and its objects are the survivor's; or it is a - member of a group backup that still exists, which deleting it would - tear. + PreconditionError: it is the predecessor of a backup that is not yet + completed (nothing to merge it into yet); it is a member of a + group backup that still exists (delete the group backup + instead); it is not completed itself; or the chain it belongs to + is already claimed by another merge, delete, or an active + restore (``DBController.claim_backup_chain_op``). """ backup = db_controller.get_backup_by_id(backup_id) - dependents = [b.uuid for b in db_controller.get_backups_by_lvol_id(backup.lvol_id) - if b.prev_backup_id == backup.uuid] - if dependents: - raise PreconditionError( - f"backup {backup.uuid} is the predecessor of " - f"{', '.join(d[:8] for d in dependents)}; merge it into its " - "successor instead of deleting it") + if backup.group_backup_id: try: owner = db_controller.get_group_backup_by_id(backup.group_backup_id) @@ -1038,21 +1144,41 @@ def delete_backup(backup_id): f"backup {backup.uuid} is a member of group backup " f"{backup.group_backup_id}; delete the group backup instead") - _cleanup_backup_kms_keys([backup]) - if backup.status == Backup.STATUS_COMPLETED: - try: - snode = db_controller.get_storage_node_by_id(backup.node_id) - except KeyError: - snode = None - if snode is not None and snode.status == StorageNode.STATUS_ONLINE: - try: - snode.rpc_client().bdev_lvol_s3_delete([backup.s3_id]) - except RPCException as e: - # Best effort, as delete_backups: the data plane does not - # implement the S3 delete yet, and the record goes regardless. - logger.error(f"Error deleting S3 objects of backup {backup.uuid}: {e}") - backup_events.backup_deleted(backup.cluster_id, backup.node_id, backup) - backup.remove(db_controller.kv_store) + if backup.status == Backup.STATUS_DELETING: + # Resuming a delete that claimed this backup but did not finish + # queuing its task (the process died in between): re-queue rather + # than re-claim, which would see this backup's own DELETING status + # among its siblings and refuse itself. + tasks_controller.add_backup_delete_task(backup.cluster_id, backup.node_id, [backup.uuid]) + return + + if backup.status == Backup.STATUS_FAILED: + # Uploaded nothing restorable, so there is no manifest and nothing + # in S3 to claim or wait on -- the same direct removal + # delete_backups gives every FAILED backup of a volume. + cleanup_backup_kms_keys([backup]) + backup_events.backup_deleted(backup.cluster_id, backup.node_id, backup) + backup.remove(db_controller.kv_store) + return + + successor = SuccessorIndex.of_lvol( + db_controller.get_backups_by_lvol_id(backup.lvol_id)).successor_of(backup.uuid) + if successor is not None: + if successor.status != Backup.STATUS_COMPLETED: + raise PreconditionError( + f"backup {backup.uuid} is the predecessor of {successor.uuid}, " + "which is not completed yet; it cannot be merged or deleted") + ok, reason = trigger_merge(successor, backup) + if not ok: + raise PreconditionError(f"Cannot delete backup {backup.uuid}: {reason}") + return + + if backup.status != Backup.STATUS_COMPLETED: + raise PreconditionError(f"backup {backup.uuid} is {backup.status}, not completed") + + ok, reason = _start_backup_delete([backup]) + if not ok: + raise PreconditionError(f"Cannot delete backup {backup.uuid}: {reason}") def list_backups(cluster_id=None, show_all=False): @@ -1199,6 +1325,16 @@ def import_backups(export: backup_manifest.BackupExport, cluster_id=None) -> int for manifest in group.manifests: backup_id = manifest.backup_id + # A manifest a delete has marked DELETING describes objects that + # may already be gone -- the single-backup analog of + # import_group_backup's refusal of a non-COMMITTED group + # manifest. Explicit, named targets are refused outright here; + # import_from_bucket is where a bulk import skips these instead. + if manifest.state != backup_manifest.BackupManifestState.ACTIVE: + raise PreconditionError( + f"Backup {backup_id} is {manifest.state} in its bucket, " + f"not active; it cannot be imported") + if backup_id in pending: raise ValueError(f"Backup {backup_id} is listed more than once") @@ -1256,37 +1392,214 @@ def import_backups(export: backup_manifest.BackupExport, cluster_id=None) -> int def import_from_bucket(config: BackupConfig, cluster_id=None) -> int: """Import every backup found in a bucket. + A bucket holds backups being deleted on their source cluster too; they + are not part of what it can restore, so they are left, not refused -- + the single-backup analog of ``import_group_backup_from_bucket`` leaving a + non-``COMMITTED`` group manifest in place. A backup whose ancestor was + left this way is itself refused by the chain check in `import_backups`, + which is the correct outcome: its chain is incomplete in this bucket for + as long as the ancestor is being deleted. + Raises: ManifestError: the bucket could not be read. PreconditionError: the manifests it holds cannot be imported as a batch. """ + active = [] + for manifest in discover_backups(config): + if manifest.state != backup_manifest.BackupManifestState.ACTIVE: + logger.info("Backup %s is %s in the bucket; not importing it", + manifest.backup_id, manifest.state) + continue + active.append(manifest) + return import_backups( backup_manifest.BackupExport(groups=[ backup_manifest.LocatedManifests( location=config.location(), - manifests=discover_backups(config)), + manifests=active), ]), cluster_id=cluster_id) +#: Backup statuses whose manifest this function checks, and whose manifest +#: --repair can re-derive from the record. MERGED and MERGING are excluded: +#: the merge path does not mark a manifest state at all (it writes the +#: survivor's and deletes the old one only once the merge itself is done), +#: so neither a MERGED backup's manifest existing nor its absence is a +#: reliable signal here -- that is a gap in the merge path itself, not one +#: this check can resolve by rewriting anything. +_MANIFEST_CHECKED_STATUSES = (Backup.STATUS_COMPLETED, Backup.STATUS_DELETING) + + +def check_backups(cluster_id=None, repair=False) -> bool: + """Verify every backup's FDB record agrees with its S3 manifest, in both + directions, and report the drift. The backup analog of + ``cluster_ops.check_indices``. + + For ``cluster_id`` (or every cluster with a usable backup configuration, + deployment-wide): + + * every COMPLETED or DELETING backup must have a manifest, whose + ``state`` matches (ACTIVE for COMPLETED, DELETING for DELETING) and + whose ``s3_id``/``prev_backup_id`` agree with the record; + * every ACTIVE manifest the bucket holds must name a backup record that + still exists in some status -- one that does not is an orphan, + reported but never repaired: it may be a disaster-recovery backup + nothing has imported yet, which looks identical to genuine drift from + here, and there is no record to derive a fix from either way. + + ``--repair`` rewrites a missing or disagreeing manifest from the record, + which is today's single source of truth: the project's own roadmap is to + make the bucket canonical instead (see ``controller.build_manifest``), + but that is not the status quo this function can assume. + + Returns ``True`` if nothing is left unresolved. + """ + clusters = ( + [db_controller.get_cluster_by_id(cluster_id)] + if cluster_id else db_controller.get_clusters()) + + problems = 0 + repaired = 0 + orphans = 0 + + for cluster in clusters: + try: + config = cluster.get_backup_config() + except ValueError: + continue # nothing of its own to check + + all_backups = db_controller.get_backups(cluster.get_id()) + known_s3_ids = set() + for backup in all_backups: + try: + if backup.get_location() == config.location(): + known_s3_ids.add(backup.s3_id) + except ValueError: + continue + + for backup in all_backups: + if backup.status not in _MANIFEST_CHECKED_STATUSES: + continue + try: + if backup.get_location() != config.location(): + continue # lives in a bucket this cluster does not own + except ValueError: + continue + + expected_state = (backup_manifest.BackupManifestState.DELETING + if backup.status == Backup.STATUS_DELETING + else backup_manifest.BackupManifestState.ACTIVE) + manifest = backup_manifest.read_if_present(config, UUID(backup.uuid)) + + if manifest is None: + problems += 1 + logger.error("Backup %s (%s) has no manifest in %s", + backup.uuid, backup.status, config.bucket_name) + if repair and _rewrite_manifest_from_record(backup): + repaired += 1 + continue + + disagreements = [] + if manifest.state != expected_state: + disagreements.append(f"state={manifest.state}, expected {expected_state}") + if manifest.s3_id != backup.s3_id: + disagreements.append(f"s3_id={manifest.s3_id}, expected {backup.s3_id}") + manifest_prev = str(manifest.prev_backup_id) if manifest.prev_backup_id else "" + if manifest_prev != backup.prev_backup_id: + disagreements.append( + f"prev_backup_id={manifest_prev or '(none)'}, " + f"expected {backup.prev_backup_id or '(none)'}") + if not disagreements: + continue + + problems += 1 + logger.error("Backup %s manifest disagrees with its record: %s", + backup.uuid, "; ".join(disagreements)) + if repair and _rewrite_manifest_from_record(backup): + repaired += 1 + + try: + manifests = discover_backups(config) + except backup_manifest.ManifestError as e: + problems += 1 + logger.error("Could not list bucket %s for cluster %s: %s", + config.bucket_name, cluster.get_id(), e) + continue + + for manifest in manifests: + if manifest.state != backup_manifest.BackupManifestState.ACTIVE: + continue # a delete's own manifest; may briefly outlive its record + if manifest.s3_id not in known_s3_ids: + orphans += 1 + logger.error( + "Manifest for backup %s (s3_id=%s) in %s names no known " + "record in cluster %s", manifest.backup_id, manifest.s3_id, + config.bucket_name, cluster.get_id()) + + logger.info("Backup check: %d problem(s), %d repaired, %d orphaned manifest(s) " + "(never auto-repaired)", problems, repaired, orphans) + if problems > repaired and not repair: + logger.error("Re-run with --repair to rewrite manifests from their records.") + return (problems - repaired) == 0 and orphans == 0 + + +def _rewrite_manifest_from_record(backup: Backup) -> bool: + """Rewrite ``backup``'s manifest from its FDB record: ACTIVE, then + DELETING again if that is what the record says. Two steps because + ``write_manifest`` always writes ACTIVE (it is what a freshly completed + backup's manifest has never not been), and only a second, explicit mark + turns that into DELETING -- the same two calls ``_start_backup_delete`` + itself would have made, in the same order. + + Never raises: a record this cannot rebuild from (an unreadable location, + a missing encryption descriptor, an unreachable bucket) is one + ``check_backups`` has to report and move on from, not one that aborts + the whole run over a single bad backup. Returns whether it succeeded. + """ + try: + write_manifest(backup) + if backup.status == Backup.STATUS_DELETING: + mark_manifest_deleting(backup) + except (ValueError, backup_manifest.ManifestError, PreconditionError, KeyError) as e: + logger.error("Could not rewrite the manifest of backup %s from its " + "record: %s", backup.uuid, e) + return False + return True + + def trigger_merge(keep_backup, old_backup): """Merge ``old_backup`` into ``keep_backup``, its successor in one volume's - chain: mark the old one merging and queue the merge task. - - Both must be completed; otherwise nothing happens and a later pass retries. - Used by the single-volume retention (policy) and by the group backup's - retirement of a pruned generation's members (group_backup). + chain: claim the chain, mark the old one merging, and queue the merge task. + + Both must be completed, and the chain must not already be claimed by + another merge, delete, or an active restore + (``DBController.claim_backup_chain_op``) -- otherwise nothing happens and + a later pass retries. Used by the single-volume retention (policy), by + the group backup's retirement of a pruned generation's members + (group_backup), and by ``delete_backup`` when the backup requested for + deletion has a completed successor: deleting it and merging it into that + successor are the same operation up to this point, so both go through + here rather than duplicating the claim. + + Returns ``(True, None)`` once the merge task is queued, or + ``(False, reason)`` if nothing happened. The scheduled callers above + treat ``False`` as "try again next cycle" and ignore the reason; + ``delete_backup`` surfaces it to its own caller. """ if old_backup.status != Backup.STATUS_COMPLETED: - return + return False, f"backup {old_backup.uuid} is {old_backup.status}, not completed" if keep_backup.status != Backup.STATUS_COMPLETED: - return + return False, f"backup {keep_backup.uuid} is {keep_backup.status}, not completed" - old_backup.status = Backup.STATUS_MERGING - old_backup.write_to_db() + ok, reason = db_controller.claim_backup_chain_op( + old_backup.lvol_id, [old_backup.uuid], Backup.STATUS_MERGING) + if not ok: + return False, reason tasks_controller.add_backup_merge_task( keep_backup.cluster_id, keep_backup.node_id, keep_backup.uuid, old_backup.uuid) + return True, None diff --git a/simplyblock_core/controllers/backup/group_backup.py b/simplyblock_core/controllers/backup/group_backup.py index 695b4e55aa..bcbfc7278e 100644 --- a/simplyblock_core/controllers/backup/group_backup.py +++ b/simplyblock_core/controllers/backup/group_backup.py @@ -624,13 +624,19 @@ def _config_for_deleting(db, group_backup): def _members_touched(db, group_backup) -> bool: - """Whether retiring a member has begun: a member deleted, merging, or merged.""" + """Whether retiring a member has begun: a member deleted, merging, + merged, or claimed for deletion (DELETING -- not yet actually gone, but + its own FN_BACKUP_DELETE task may already be destroying its S3 objects, + which is exactly the in-flight state a restore back to COMPLETED must + not paper over). + """ for backup_id in group_backup.member_backup_ids: try: member = db.get_backup_by_id(backup_id) except KeyError: return True - if member.status in (Backup.STATUS_MERGING, Backup.STATUS_MERGED): + if member.status in (Backup.STATUS_MERGING, Backup.STATUS_MERGED, + Backup.STATUS_DELETING): return True return False @@ -652,8 +658,15 @@ def _retire_members(db, group_backup) -> bool: continue # deleted if member.status == Backup.STATUS_MERGED: continue + if member.status == Backup.STATUS_DELETING: + # Already claimed by an earlier pass; its own FN_BACKUP_DELETE + # task is retiring it. Not retired until that task actually + # removes the record (the KeyError branch above, on a later + # pass) -- re-claiming it here would needlessly re-queue the task. + retired = False + continue chain = db.get_backups_by_lvol_id(member.lvol_id) - successor = next((b for b in chain if b.prev_backup_id == member.uuid), None) + successor = backup_chain.SuccessorIndex.of_lvol(chain).successor_of(member.uuid) busy = (member.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS) or any(b.status == Backup.STATUS_MERGING for b in chain)) if successor is None and not busy: @@ -661,16 +674,34 @@ def _retire_members(db, group_backup) -> bool: backup_controller.delete_backup(member.uuid) except KeyError: pass # deleted concurrently - logger.info("Group backup %s: member %s deleted, nothing chains against it", - group_backup.uuid, member.uuid[:8]) + # A FAILED member is removed synchronously (nothing was ever + # uploaded for it, so there is no manifest or task to wait on); + # a COMPLETED one is only claimed -- DELETING now, actually gone + # once its own FN_BACKUP_DELETE task finishes. Checked rather + # than branched on the member's status before the call, so this + # reads the one fact that actually answers it either way. + try: + db.get_backup_by_id(member.uuid) + except KeyError: + logger.info("Group backup %s: member %s deleted, nothing " + "chains against it", group_backup.uuid, member.uuid[:8]) + else: + logger.info("Group backup %s: member %s claimed for deletion, " + "nothing chains against it", + group_backup.uuid, member.uuid[:8]) + retired = False continue retired = False if (successor is not None and not busy and member.status == Backup.STATUS_COMPLETED and successor.status == Backup.STATUS_COMPLETED): - backup_controller.trigger_merge(successor, member) - logger.info("Group backup %s: member %s merging into its successor %s", - group_backup.uuid, member.uuid[:8], successor.uuid[:8]) + ok, reason = backup_controller.trigger_merge(successor, member) + if ok: + logger.info("Group backup %s: member %s merging into its successor %s", + group_backup.uuid, member.uuid[:8], successor.uuid[:8]) + else: + logger.info("Group backup %s: member %s not merged into %s yet: %s", + group_backup.uuid, member.uuid[:8], successor.uuid[:8], reason) return retired @@ -710,11 +741,11 @@ def retire_orphaned_member_backups(lvol_ids, db=None): db.get_group_backup_by_id(b.group_backup_id) except KeyError: orphans.append(b) - by_prev = {b.prev_backup_id: b for b in chain if b.prev_backup_id} + successors = backup_chain.SuccessorIndex.of_lvol(chain) # Oldest first: the base of the chain is the one whose merge unblocks # the next, and a leaf is only a leaf once nothing is below it. for old in sorted(orphans, key=lambda b: (b.created_at, b.uuid)): - successor = by_prev.get(old.uuid) + successor = successors.successor_of(old.uuid) if successor is None: try: backup_controller.delete_backup(old.uuid) diff --git a/simplyblock_core/controllers/backup/manifest.py b/simplyblock_core/controllers/backup/manifest.py index 0a2d6b6aa9..84fdb9e34d 100644 --- a/simplyblock_core/controllers/backup/manifest.py +++ b/simplyblock_core/controllers/backup/manifest.py @@ -273,6 +273,25 @@ class ManifestDataPlane(BaseModel): with_compression: bool = False +class BackupManifestState(StrEnum): + """Whether a backup manifest's objects can be relied on. + + A manifest is written (and stays) ``active`` for as long as its backup is + restorable from its own objects. A delete marks it ``deleting`` before it + touches any S3 object, and removes the manifest only once the data plane + has confirmed those objects are gone -- the single-backup half of the same + metadata-before-data ordering :class:`GroupManifestState` keeps for group + backups. A reader that is not the one deleting it -- a disaster-recovery + ``discover``/``import`` on another cluster -- must skip a manifest in this + state rather than treat it as restorable, the same rule + ``import_group_backup_from_bucket`` already applies to a non-``COMMITTED`` + group manifest. A manifest written before deletes marked it reads as + ``active``, which is what it was. + """ + ACTIVE = "active" + DELETING = "deleting" + + class BackupManifest(BaseModel): model_config = ConfigDict(extra="forbid") @@ -304,6 +323,9 @@ class BackupManifest(BaseModel): volume: ManifestVolume dataplane: ManifestDataPlane + #: See BackupManifestState. + state: BackupManifestState = BackupManifestState.ACTIVE + EXPORT_SCHEMA_VERSION = 1 @@ -345,6 +367,7 @@ def s3_client(config: BackupConfig): Credentials are passed only when configured; omitting them lets boto3 fall back to its default provider chain (instance IAM role, environment, profile), which is what an absent ``credentials`` means. + """ return boto3.client("s3", region_name=config.region, @@ -437,8 +460,9 @@ def delete(config: BackupConfig, backup_id: UUID) -> None: """Remove a manifest. Only for a backup whose data is genuinely gone -- a merge folding it into - its successor. Deleting the manifest of a backup whose objects still exist - turns them into weight nobody can identify or reclaim. + its successor, or a delete that has confirmed the data plane removed its + objects. Deleting the manifest of a backup whose objects still exist turns + them into weight nobody can identify or reclaim. """ try: s3_client(config).delete_object( @@ -447,6 +471,100 @@ def delete(config: BackupConfig, backup_id: UUID) -> None: raise ManifestError(f"Failed to delete manifest for backup {backup_id}") from e +def delete_data_objects(config: BackupConfig, s3_id: int) -> None: + """Remove every object a backup's ``s3_id`` owns under ``key_format`` + (``{s3_id}/{mid}/{extent}``) -- its data clusters and its metadata object. + + Only for a backup whose deletion needs no data-plane involvement: a tip + delete or a whole-chain delete, where nothing else names these objects. + A backup something still chains against is folded into its successor by + the data plane instead (``bdev_lvol_s3_merge``), which removes the old + backup's objects itself as part of the merge -- this function is never + called for that case. + + Idempotent: an ``s3_id`` prefix with nothing left under it lists empty and + this is a no-op, so retrying after a partial failure costs one empty list + call and nothing else. + + One ``delete_object`` per key rather than batching through + ``delete_objects``: that call's multi-object delete requires a legacy + Content-MD5 header botocore (1.36+) no longer computes by default, in + favour of the newer flexible checksums -- confirmed live against MinIO + (RELEASE.2024-01-16), which does not understand those and answers + ``MissingContentMD5``, including with ``request_checksum_calculation`` + set back to ``"when_required"``. Single-object delete carries no such + requirement and needs no workaround. + + Raises: + ManifestError: the bucket could not be listed, or an object under this + prefix could not be removed. + """ + client = s3_client(config) + prefix = f"{s3_id}/" + try: + pages = client.get_paginator("list_objects_v2").paginate( + Bucket=config.bucket_name, Prefix=prefix) + for page in pages: + for entry in page.get("Contents", []): + client.delete_object(Bucket=config.bucket_name, Key=entry["Key"]) + except (BotoCoreError, ClientError) as e: + raise ManifestError( + f"Failed to delete data objects under {prefix} in {config.bucket_name}") from e + + +def read_if_present(config: BackupConfig, backup_id: UUID) -> BackupManifest | None: + """:func:`read`, but ``None`` for a manifest the bucket does not hold.""" + try: + body = s3_client(config).get_object( + Bucket=config.bucket_name, Key=manifest_key(backup_id))["Body"].read() + except ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + return None + raise ManifestError( + f"Failed to read manifest for backup {backup_id} " + f"from {config.bucket_name}") from e + except BotoCoreError as e: + raise ManifestError( + f"Failed to read manifest for backup {backup_id} " + f"from {config.bucket_name}") from e + return _parse(body, manifest_key(backup_id)) + + +def mark_deleting(config: BackupConfig, backup_id: UUID) -> None: + """Mark a backup's manifest DELETING, before any of its S3 objects are + touched -- the single-backup half of the metadata-before-data ordering + :func:`delete_group_backup ` keeps for + group backups. + + A no-op if the manifest already says so (a delete resuming after a + restart or retry), or if it is already gone (a delete that got as far as + removing the manifest before being abandoned -- :func:`delete`, not this, + is the next step on retry, not this one again). + + Raises: + ManifestError: the manifest could not be read or rewritten. + """ + manifest = read_if_present(config, backup_id) + if manifest is not None and manifest.state != BackupManifestState.DELETING: + write(config, manifest.model_copy(update={"state": BackupManifestState.DELETING})) + + +def mark_active(config: BackupConfig, backup_id: UUID) -> None: + """Revert a backup's manifest to ACTIVE: a delete that did not finish. + + A no-op if the manifest is already active, or already gone (the delete + got as far as removing it before being abandoned -- there is nothing left + to revert, and the FDB record it was abandoned alongside is reverted + separately). + + Raises: + ManifestError: the manifest could not be read or rewritten. + """ + manifest = read_if_present(config, backup_id) + if manifest is not None and manifest.state != BackupManifestState.ACTIVE: + write(config, manifest.model_copy(update={"state": BackupManifestState.ACTIVE})) + + def _parse(body: bytes, key: str) -> BackupManifest: try: data = json.loads(body) diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index e9c7dad063..5b8ac415f0 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -1578,10 +1578,18 @@ def add_group_backup_commit_task(group_backup): ) -def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids, - lvol_id="", s3_config=None): +def add_backup_restore_task(source_lvol_id, cluster_id, node_id, backup_id, lvol_name, + chain_ids, lvol_id="", s3_config=None): """Create the task that restores an S3 backup chain into a new lvol. + Unlike every other task type, this does not go through ``_add_task``: its + creation has to be atomic with checking that no backup of the SOURCE + chain (``source_lvol_id`` -- the volume the backup was taken from, not + ``lvol_id``, the new volume being restored into) is already + ``MERGING``/``DELETING``, so a delete or merge cannot slip into the gap + between that check and the task actually existing. See + ``DBController.create_backup_restore_task_if_chain_clear``. + Args: s3_config: set when the backup lives in a bucket that is not the cluster's own, so the runner has to attach a device of its own to @@ -1589,13 +1597,12 @@ def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids and deletes it -- scrubbing this field -- once the restore reaches a terminal state. Unset, the restore reads through the device the node already has for the cluster's own bucket. + + Returns the task uuid, or ``False`` if the source chain is being merged + or deleted. """ - return _add_task( - JobSchedule.FN_BACKUP_RESTORE, - cluster_id, - node_id, - "", - max_retry=constants.BACKUP_MAX_RETRIES, + task, reason = db.create_backup_restore_task_if_chain_clear( + source_lvol_id, cluster_id, node_id, function_params={ "backup_id": backup_id, "lvol_name": lvol_name, @@ -1603,6 +1610,40 @@ def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids "chain_ids": chain_ids, "s3_config": s3_config, }, + max_retry=constants.BACKUP_MAX_RETRIES, + ) + if task is None: + logger.info(f"Cannot create restore task: {reason}") + return False + tasks_events.task_create(task) + return task.uuid + + +def add_backup_delete_task(cluster_id, node_id, backup_ids): + """Create the task that deletes one or more backups of the same volume + that nothing chains against, unless an open task already covers them. + + The targets are already DELETING by the time this is called -- + ``backup_controller.delete_backup``/``delete_backups`` claim them via + ``DBController.claim_backup_chain_op`` first -- so, like + ``add_backup_merge_task``, this needs no transactional join of its own: + there is no gap between a check and this task existing left to close. + """ + wanted = set(backup_ids) + for task in db.get_job_tasks_by_function(cluster_id, JobSchedule.FN_BACKUP_DELETE): + if (not task.canceled and task.status != JobSchedule.STATUS_DONE + and wanted <= set((task.function_params or {}).get("backup_ids") or [])): + logger.info(f"Task found, skip adding new task: {task.uuid}") + return False + return _add_task( + JobSchedule.FN_BACKUP_DELETE, + cluster_id, + node_id, + "", + max_retry=constants.BACKUP_TASK_MAX_RETRIES, + function_params={ + "backup_ids": list(backup_ids), + }, ) diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index d8a96c85b2..6119cd8434 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -5,6 +5,7 @@ import os.path import struct import time +import uuid from typing import Any, ClassVar import fdb @@ -1489,6 +1490,176 @@ def release_backup_chain_locks(self, snapshot_ids): transactional = fdb.transactional(DBController._release_backup_chain_locks_tx) transactional(self, self.kv_store, ordered_snapshot_ids) + # ---- Backup chain operation guard (Single FDB Transaction) ---- + # + # Mutual exclusion between a delete, a merge and a restore on the same + # volume's backup chain. Unlike acquire_backup_chain_locks above (which + # only protects the creation-time enqueue window), this guards the whole + # of a destructive operation -- but it does so WITHOUT holding anything + # open: the chain is bounded (constants.BACKUP_MAX_CHAIN_LENGTH), so + # reading every sibling and flipping the target(s) fits in one + # transaction, and FDB's serializable isolation is what makes two + # concurrent claims over the same chain mutually exclusive (see + # simplyblock_core/AGENTS.md: a critical section that fits in one + # transaction needs no separate lock, distributed or otherwise). The + # status this writes -- MERGING or DELETING -- IS the lock for everything + # after the claim: every other caller refuses to touch a chain holding + # either, which is what makes a long-running merge/delete task safe + # without this transaction, or anything else, staying open for its + # duration. + + def _claim_backup_chain_op_tx(self, tr, lvol_id, target_ids, new_status): + idx = indices.get_index(Backup, 'lvol_id') + sibling_ids = [ + idx.entry_id(Backup, bytes(key), bytes(value)) + for key, value in tr.get_range_startswith(idx.prefix(Backup, (lvol_id,))) + ] + # Keyed by the bare uuid, not get_id(): every caller holds a Backup's + # uuid, never its "/" composite id, and a chain + # never spans clusters -- so the uuid alone is already unambiguous + # within one lvol_id's siblings. + siblings = {} + for sibling_id in sibling_ids: + rows = Backup().read_from_db(tr, id=sibling_id) + if rows: + siblings[rows[0].uuid] = rows[0] + + target_set = set(target_ids) + # A sibling already MERGING/DELETING blocks the claim UNLESS it is one + # of this call's own targets AND already sitting in new_status: that + # is what makes resuming an interrupted delete/merge (the caller is + # retrying, not racing itself) safe. A target held in the OTHER + # status -- claimed by a genuinely different operation -- still + # blocks, same as any other sibling. + blocking = next((b for b in siblings.values() + if b.status in (Backup.STATUS_MERGING, Backup.STATUS_DELETING) + and not (b.uuid in target_set and b.status == new_status)), None) + if blocking is not None: + return False, f"backup {blocking.uuid} of this chain is already {blocking.status}" + + restore_idx = indices.get_index(JobSchedule, 'restore_chain_s3_id') + for backup in siblings.values(): + restoring = any(True for _ in tr.get_range_startswith( + restore_idx.prefix(JobSchedule, (backup.s3_id,)), limit=1)) + if restoring: + return False, f"backup {backup.uuid} is being read by an active restore" + + missing = [t for t in target_ids if t not in siblings] + if missing: + return False, (f"backup(s) {', '.join(missing)} not found in volume " + f"{lvol_id}'s chain") + + index_list = Backup.active_indexes(tr) + for target_id in target_ids: + target = siblings[target_id] + old_keys = BaseModel.index_keys(Backup, index_list, target) + target.status = new_status + target.updated_at = str(datetime.datetime.now(datetime.UTC)) + if index_list: + BaseModel._apply_index_diff(tr, Backup, index_list, old_keys, target) + tr[target.get_db_id().encode()] = json.dumps(target.to_dict(unwrap_secrets=True)).encode() + scope = target.watch_scope() + tr.add(watches.watch_index_rollup_key(Backup, scope), watches.ONE_LE64) + tr.add(watches.watch_index_version_key(Backup, scope, target.get_id()), watches.ONE_LE64) + + return True, None + + def claim_backup_chain_op(self, lvol_id, target_ids, new_status): + """Atomically claim one or more backups of ``lvol_id``'s chain (named + by their bare ``uuid``, not ``get_id()``) for a destructive operation + (merge or delete). + + Refuses, as ``(False, reason)``, if any backup of the chain is already + ``MERGING``/``DELETING`` under a status other than ``new_status`` -- + whether or not it is one of ``target_ids`` -- or if any backup in the + chain (target or not) is named by an active restore's own chain + (``JobSchedule.restore_chain_s3_id``). On success, every id in + ``target_ids`` is flipped to ``new_status`` and ``(True, None)`` is + returned. + + Idempotent by design, not exclusive between identical claims: a target + already in ``new_status`` -- this call resuming one that claimed it + but did not finish queuing the task that follows -- does not block + itself. Two callers racing to claim the very same ids at once can + therefore BOTH succeed (one commits, the other retries on conflict, + re-reads the now-already-``new_status`` targets, and finds nothing of + its own to object to) -- what this guarantees is that no OTHER, + genuinely different claim can interleave with either, not that only + one of two identical ones wins. Callers rely on idempotency further + downstream (``manifest.mark_deleting`` is a no-op if already marked; + ``add_backup_delete_task`` dedupes against an open task) to make a + repeat claim of the same ids harmless rather than merely admitted. + + Assumes ``Backup``'s ``lvol_id`` index is ready, which it already is in + any deployment old enough to run this: an index still ``building`` would + make this read incomplete rather than merely slow, so this does not fall + back to a scan the way a read through ``query()`` would. + """ + if not self.kv_store or not target_ids: + return False, "No DB connection" if not self.kv_store else "No targets given" + transactional = fdb.transactional(DBController._claim_backup_chain_op_tx) + return transactional(self, self.kv_store, lvol_id, list(target_ids), new_status) + + def _create_backup_restore_task_tx(self, tr, source_lvol_id, cluster_id, node_id, + function_params, max_retry, task_uuid, task_date): + idx = indices.get_index(Backup, 'lvol_id') + sibling_ids = [ + idx.entry_id(Backup, bytes(key), bytes(value)) + for key, value in tr.get_range_startswith(idx.prefix(Backup, (source_lvol_id,))) + ] + for sibling_id in sibling_ids: + rows = Backup().read_from_db(tr, id=sibling_id) + if rows and rows[0].status in (Backup.STATUS_MERGING, Backup.STATUS_DELETING): + return None, f"backup {rows[0].uuid} of this chain is {rows[0].status}" + + task_obj = JobSchedule() + task_obj.uuid = task_uuid + task_obj.cluster_id = cluster_id + task_obj.node_id = node_id + task_obj.date = task_date + task_obj.function_name = JobSchedule.FN_BACKUP_RESTORE + task_obj.function_params = function_params + task_obj.max_retry = max_retry + task_obj.status = JobSchedule.STATUS_NEW + + index_list = JobSchedule.active_indexes(tr) + if index_list: + BaseModel._apply_index_diff( + tr, JobSchedule, index_list, BaseModel.index_keys(JobSchedule, index_list, None), + task_obj) + tr[task_obj.get_db_id().encode()] = json.dumps(task_obj.to_dict(unwrap_secrets=True)).encode() + scope = task_obj.watch_scope() + tr.add(watches.watch_index_rollup_key(JobSchedule, scope), watches.ONE_LE64) + tr.add(watches.watch_index_version_key(JobSchedule, scope, task_obj.get_id()), watches.ONE_LE64) + + return task_obj, None + + def create_backup_restore_task_if_chain_clear(self, source_lvol_id, cluster_id, node_id, + function_params, max_retry): + """Create the FN_BACKUP_RESTORE task that drives a restore, atomically + with checking that no backup of ``source_lvol_id``'s chain is already + ``MERGING``/``DELETING``. + + Joining the check and the task's creation in one transaction is what + closes the gap a plain ``chain.require_restorable()`` followed by a + separate ``write_to_db()`` would leave open: without it, a delete or + merge could claim the chain (``claim_backup_chain_op``) in between, + and this task would start reading objects that operation is about to + remove. The task itself, once created, is what ``claim_backup_chain_op`` + then sees (via the ``restore_chain_s3_id`` index) and refuses to let a + later delete or merge touch -- see that method and the index's + declaration on ``JobSchedule``. + + Returns ``(task, None)`` on success, or ``(None, reason)`` if refused. + """ + if not self.kv_store: + return None, "No DB connection" + task_uuid = str(uuid.uuid4()) + task_date = int(time.time()) + transactional = fdb.transactional(DBController._create_backup_restore_task_tx) + return transactional(self, self.kv_store, source_lvol_id, cluster_id, node_id, + function_params, max_retry, task_uuid, task_date) + # ---- Per-lvstore snapshot-mutation lock (Single FDB Transaction) ---- def _try_acquire_lvstore_lock_tx(self, tr, cluster_id, lvs_name, owner, now, ttl): @@ -2079,9 +2250,10 @@ def next_vuid(self) -> int: # old allocator was max-plus-one over the local cluster's Backup records, # which had three problems: it raced (two concurrent backups got the same # id), it recycled the id of a deleted backup whose objects may still exist - # (nothing reclaims them -- bdev_lvol_s3_delete does not exist on the data - # plane), and after an import it counted foreign backups it should not have. - # A monotonic sequence makes reuse impossible by construction. + # (deletion removes them, but not atomically with the record disappearing, + # so a window exists where the old id's objects are still reachable), and + # after an import it counted foreign backups it should not have. A + # monotonic sequence makes reuse impossible by construction. # # Unlike vuid this space is NOT unbounded: the data plane packs s3_id into # 30 bits and masks rather than validates, so callers must check diff --git a/simplyblock_core/models/job_schedule.py b/simplyblock_core/models/job_schedule.py index 31d729421d..42c6ff7482 100644 --- a/simplyblock_core/models/job_schedule.py +++ b/simplyblock_core/models/job_schedule.py @@ -35,6 +35,22 @@ class JobSchedule(BaseModel): and t.function_params.get('snapshot_id') else [] )), + # Every backup s3_id a non-terminal FN_BACKUP_RESTORE task is reading + # (its chain_ids -- see FN_BACKUP_RESTORE below -- are s3_ids, not + # backup uuids). db_controller.claim_backup_chain_op reads this before + # letting a delete or merge touch a chain, so a restore in flight blocks + # them. Self-maintaining: the extractor yields nothing once the task + # reaches STATUS_DONE or is canceled, so the entry disappears in the + # same transaction as that write -- nothing decrements it by hand, and + # nothing can leak it. + Index('restore_chain_s3_id', arity=1, extract=lambda t: ( + [(s3_id,) for s3_id in (t.function_params.get('chain_ids') or [])] + if t.function_name == t.FN_BACKUP_RESTORE + and isinstance(t.function_params, dict) + and t.status != t.STATUS_DONE + and not t.canceled + else [] + )), ) STATUS_NEW = 'new' @@ -64,6 +80,11 @@ class JobSchedule(BaseModel): FN_BACKUP = "s3_backup" FN_BACKUP_RESTORE = "s3_backup_restore" FN_BACKUP_MERGE = "s3_backup_merge" + # Delete one or more backups of the same volume that nothing chains + # against: remove their S3 objects, then their manifests, then their + # records -- metadata-before-data on the way in (claim_backup_chain_op + # marks them DELETING first), data-before-metadata on the way out. + FN_BACKUP_DELETE = "s3_backup_delete" # Consistency-group backup commit: watch the member backups of one group # generation and publish the group manifest once every member completed, # the all-or-nothing barrier. diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 467434496f..35252a919d 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -2784,10 +2784,5 @@ def bdev_s3_delete(self, name: str): """ return self._request("bdev_s3_delete", name=name) - def bdev_lvol_s3_delete(self, s3_ids: list[int]): - """Delete all S3 backups for the given IDs (list of uint32).""" - # RPC still missing on data plane — use dummy - return self._request("bdev_lvol_s3_delete", s3_ids=s3_ids) - def bdev_nvme_get_controller_health_info(self, name): return self._request("bdev_nvme_get_controller_health_info", name=name) diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index d0f14b0a7e..5a09cee06f 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -414,6 +414,66 @@ def _run_merge(task): raise TaskDefer("Merge in progress") +def _run_delete(task): + """Delete the backups named by ``function_params["backup_ids"]``: remove + their S3 objects, then their manifests, then their records -- the reverse + of the order ``backup_controller._start_backup_delete`` claimed them in + (record marked DELETING, then manifest marked DELETING, then this task). + + A backup already gone, or no longer DELETING, is skipped: a prior cycle + of this same task already retired it, or (the group-backup path) a + different caller did. + + No data-plane RPC: there is no ``bdev_lvol_s3_delete`` and none is + planned, so the control plane removes the objects itself, the same way it + removes a manifest. This handler is only ever reached for a backup nothing + chains against -- one with a successor is folded into it by + ``_run_merge`` instead, which is where the data plane stays involved + (``bdev_lvol_s3_merge``). Single-cycle like the RPC path it replaces: S3 + delete is synchronous from the caller's side, with nothing to poll. + """ + backup_ids = task.function_params.get("backup_ids") or [] + if not backup_ids: + raise TaskAbort("Missing backup_ids") + + backups = [] + for backup_id in backup_ids: + try: + backup = db.get_backup_by_id(backup_id) + except KeyError: + continue + if backup.status != Backup.STATUS_DELETING: + continue + backups.append(backup) + + if not backups: + set_result(task, "Nothing left to delete") + return + + for backup in backups: + try: + backup_controller.delete_backup_data(backup) + except (ManifestError, PreconditionError) as e: + raise TaskRetry(f"S3 data delete failed: {e}") + + # The data is confirmed gone: metadata follows data on the way out, the + # reverse of how the claim marked intent before any of this ran. + for backup in backups: + try: + backup_controller.delete_manifest(backup) + except (ManifestError, PreconditionError) as e: + # The S3 delete already happened and is not reversible, so the + # task cannot be aborted here -- retry the manifest/record cleanup. + raise TaskRetry(f"Delete done, manifest cleanup failed: {e}") + + backup_controller.cleanup_backup_kms_keys([backup]) + backup_events.backup_deleted(backup.cluster_id, backup.node_id, backup) + backup.remove(db.kv_store) + + set_result(task, "Backups deleted") + logger.info(f"Deleted backups: {', '.join(b.uuid for b in backups)}") + + def _run_group_backup_commit(task): """Commit a consistency-group backup once its members are done. @@ -468,6 +528,7 @@ def _run_group_backup_delete(task): JobSchedule.FN_BACKUP: _run_backup, JobSchedule.FN_BACKUP_RESTORE: _run_restore, JobSchedule.FN_BACKUP_MERGE: _run_merge, + JobSchedule.FN_BACKUP_DELETE: _run_delete, JobSchedule.FN_GROUP_BACKUP_COMMIT: _run_group_backup_commit, JobSchedule.FN_GROUP_BACKUP_DELETE: _run_group_backup_delete, } @@ -558,6 +619,31 @@ def finalize_resource(task): gb.error_message = reason gb.write_to_db() + elif task.function_name == JobSchedule.FN_BACKUP_DELETE: + # The delete did not finish (timeout, retry ceiling, abort, + # cancellation): every backup this task still finds DELETING had its + # S3 objects left untouched -- the delete either never ran or never + # confirmed success, and _run_delete only removes a manifest/record + # after it has -- so each is reverted to COMPLETED, record and + # manifest both, the same way an unfinished merge leaves its old + # backup intact. + for backup_id in task.function_params.get("backup_ids") or []: + try: + backup = db.get_backup_by_id(backup_id) + except KeyError: + continue + if backup.status != Backup.STATUS_DELETING: + continue + try: + backup_controller.revert_manifest_deleting(backup) + except (ManifestError, PreconditionError, KeyError) as e: + logger.error( + f"Backup {backup.uuid} reverted to COMPLETED, but its " + f"manifest could not be marked active and still says " + f"deleting: {e}") + backup.status = Backup.STATUS_COMPLETED + backup.write_to_db() + elif task.function_name == JobSchedule.FN_GROUP_BACKUP_DELETE: group_backup_id = task.function_params.get("group_backup_id") if not group_backup_id: @@ -582,7 +668,8 @@ def finalize_resource(task): #: "max retry reached (8/8)" and would end any other task older than the #: backup timeout (2026-09-29). BACKUP_FUNCTIONS = (JobSchedule.FN_BACKUP, JobSchedule.FN_BACKUP_RESTORE, - JobSchedule.FN_BACKUP_MERGE, JobSchedule.FN_GROUP_BACKUP_COMMIT, + JobSchedule.FN_BACKUP_MERGE, JobSchedule.FN_BACKUP_DELETE, + JobSchedule.FN_GROUP_BACKUP_COMMIT, JobSchedule.FN_GROUP_BACKUP_DELETE) diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index c946c655bf..b33f5334ed 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -150,6 +150,7 @@ "s3_backup", "s3_backup_restore", "s3_backup_merge", + "s3_backup_delete", "group_backup_commit", "group_backup_delete", "cluster_expand", diff --git a/simplyblock_web/api/v2/cluster/backup.py b/simplyblock_web/api/v2/cluster/backup.py index aa5e123f9d..068c2a3e94 100644 --- a/simplyblock_web/api/v2/cluster/backup.py +++ b/simplyblock_web/api/v2/cluster/backup.py @@ -9,6 +9,7 @@ from simplyblock_core.controllers.backup import policy as backup_policy from simplyblock_core.controllers.backup.manifest import ManifestError from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup_config import S3Credentials from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.lvol_model import LVol @@ -173,8 +174,8 @@ def _lookup_lvol_in_cluster(volume_id: str, cluster: ClusterModel) -> LVol: @api.delete( '/{volume_id}', name='clusters:backups:delete', - status_code=204, - responses={204: {"content": None}}, + status_code=202, + responses={202: {"content": None}}, deprecated=True, summary='Deprecated — delete all backups for a volume', description=( @@ -187,8 +188,8 @@ def delete_backups(cluster: Cluster, volume_id: UUID) -> Response: volume = _lookup_lvol_in_cluster(str(volume_id), cluster) success, error = backup_controller.delete_backups(volume.get_id()) if error: - raise HTTPException(400, error) - return Response(status_code=204) + raise HTTPException(409, error) + return Response(status_code=202) # Backup policies @@ -286,4 +287,19 @@ def get_backup(cluster: Cluster, backup: BackupResource) -> BackupDTO: return BackupDTO.from_model(backup) +@instance_api.delete('/', name='clusters:backups:delete-one', + status_code=202, responses={202: {"content": None}}) +def delete_backup(cluster: Cluster, backup: BackupResource) -> Response: + """Delete one backup. A backup nothing chains against is claimed for + deletion: its record and manifest are marked deleting, and a task + removes its S3 objects and then both. A backup something chains against + is instead merged into that successor -- deleting it and merging it are + the same request once a successor exists.""" + try: + backup_controller.delete_backup(backup.uuid) + except PreconditionError as e: + raise HTTPException(409, str(e)) from e + return Response(status_code=202) + + api.include_router(instance_api) diff --git a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py index b5f1f282ae..11b9a7d4cd 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py @@ -446,14 +446,18 @@ def create_backup( @instance_api.delete( '/backups', name='clusters:storage-pools:volumes:backups:delete', - status_code=204, - responses={204: {"content": None}}, + status_code=202, + responses={202: {"content": None}}, ) def delete_backups(cluster: Cluster, pool: StoragePool, volume: Volume) -> Response: + """Start deleting every backup of this volume. Completed backups are + claimed and a task removes their S3 objects and then their records and + manifests; the request returns once they are claimed, not once they are + gone.""" success, error = backup_controller.delete_backups(volume.get_id()) if error: - raise HTTPException(400, error) - return Response(status_code=204) + raise HTTPException(409, error) + return Response(status_code=202) api.include_router(replication_collection_api) instance_api.include_router(replication_api, prefix='/replication') diff --git a/tests/integration/backup/test_check_backups.py b/tests/integration/backup/test_check_backups.py new file mode 100644 index 0000000000..56d038d5c1 --- /dev/null +++ b/tests/integration/backup/test_check_backups.py @@ -0,0 +1,200 @@ +"""Checking a backup's FDB record against its S3 manifest, against real +FoundationDB. + +The backup analog of the secondary-index checker (``check_indices``): +read-only unless ``--repair`` is given, and reports in both directions -- +a record whose manifest is missing or disagrees, and a manifest naming no +known record. Only the first direction is ever auto-repaired; an orphaned +manifest may be a disaster-recovery backup nothing has imported yet, which +looks identical to genuine drift from here. + +The database is real. The bucket is the one boundary faked, as an in-memory +one, so the manifest functions under test run for real over it. +""" +import io +import zlib +from uuid import UUID + +import pytest +from botocore.exceptions import ClientError + +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import ( + BackupManifest, + BackupManifestState, + ManifestDataPlane, + ManifestSource, + ManifestVolume, +) +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster + + +def _uuid(name: str) -> str: + return f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000" + + +CLUSTER = _uuid("cluster") +OTHER_CLUSTER = _uuid("other-cluster") +LVOL = _uuid("lvol") +NODE = _uuid("node") +BUCKET = "simplyblock-backup-cluster" + + +def _config(): + return BackupConfig.model_validate({"bucket_name": BUCKET, "region": "eu-central-1"}) + + +@pytest.fixture +def db(): + db = DBController() + for cluster_id in (CLUSTER, OTHER_CLUSTER): + c = Cluster() + c.uuid = cluster_id + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return db + + +class _Bucket: + def __init__(self): + self.objects: dict = {} + + def put_object(self, Bucket, Key, Body, **_): + self.objects[Key] = Body + + def get_object(self, Bucket, Key): + if Key not in self.objects: + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") + return {"Body": io.BytesIO(self.objects[Key])} + + def delete_object(self, Bucket, Key): + self.objects.pop(Key, None) + + def get_paginator(self, _name): + objects = self.objects + + class _Paginator: + def paginate(self, Bucket, Prefix): + yield {"Contents": [{"Key": k} for k in objects if k.startswith(Prefix)]} + return _Paginator() + + +@pytest.fixture +def bucket(monkeypatch): + b = _Bucket() + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + return b + + +def _manifest(backup_id: str, prev: str = "", state=BackupManifestState.ACTIVE, + s3_id: int | None = None) -> BackupManifest: + # s3_id defaults to a value derived from backup_id itself -- fine for an + # orphan manifest naming no real backup, but a manifest meant to describe + # a real one MUST pass that backup's own s3_id: a Backup's s3_id is + # whatever was assigned at creation, not derivable from its uuid. + return BackupManifest( + backup_id=UUID(backup_id), + s3_id=s3_id if s3_id is not None else zlib.crc32(backup_id.encode()) % 1000, + created_at=100, + completed_at=200, + size=4096, + prev_backup_id=UUID(prev) if prev else None, + source=ManifestSource(cluster_id=UUID(CLUSTER), node_id=UUID(NODE)), + volume=ManifestVolume( + lvol_id=UUID(LVOL), lvol_name="vol", snapshot_id=UUID(_uuid("snap")), + snapshot_name="snap", size=4096), + dataplane=ManifestDataPlane(), + state=state, + ) + + +def _backup(db, name, prev="", status=Backup.STATUS_COMPLETED, cluster_id=CLUSTER): + b = Backup() + b.uuid = _uuid(name) + b.cluster_id = cluster_id + b.lvol_id = LVOL + b.lvol_name = "vol" + # build_manifest (the --repair path) needs a valid snapshot_id: a real + # COMPLETED backup always has one. + b.snapshot_id = _uuid(f"snap-{name}") + b.snapshot_name = "snap" + b.node_id = NODE + b.size = 4096 + b.s3_id = zlib.crc32(name.encode()) % 1000 + b.prev_backup_id = _uuid(prev) if prev else "" + b.status = status + b.location = _config().location().model_dump(mode="json") + b.write_to_db(db.kv_store) + return b + + +def test_a_consistent_completed_backup_passes(db, bucket): + b = _backup(db, "leaf") + backup_manifest.write(_config(), _manifest(b.uuid, s3_id=b.s3_id)) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is True + + +def test_a_completed_backup_with_no_manifest_fails_and_repairs(db, bucket): + b = _backup(db, "leaf") + + assert backup_controller.check_backups(cluster_id=CLUSTER) is False + + assert backup_controller.check_backups(cluster_id=CLUSTER, repair=True) is True + assert backup_manifest.read(_config(), UUID(b.uuid)).state == BackupManifestState.ACTIVE + # A second run finds it already fixed. + assert backup_controller.check_backups(cluster_id=CLUSTER) is True + + +def test_a_disagreeing_manifest_fails_and_repairs(db, bucket): + b = _backup(db, "leaf", prev="base") + # Wrong prev_backup_id in the manifest -- as if a merge updated the + # record but not (yet) the bucket. + backup_manifest.write(_config(), _manifest(b.uuid, prev="", s3_id=b.s3_id)) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is False + + assert backup_controller.check_backups(cluster_id=CLUSTER, repair=True) is True + assert backup_manifest.read(_config(), UUID(b.uuid)).prev_backup_id == UUID(_uuid("base")) + + +def test_a_deleting_backup_with_an_active_manifest_fails_and_repairs(db, bucket): + b = _backup(db, "leaf", status=Backup.STATUS_DELETING) + backup_manifest.write(_config(), _manifest(b.uuid, state=BackupManifestState.ACTIVE, s3_id=b.s3_id)) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is False + + assert backup_controller.check_backups(cluster_id=CLUSTER, repair=True) is True + assert backup_manifest.read(_config(), UUID(b.uuid)).state == BackupManifestState.DELETING + + +def test_an_orphaned_manifest_fails_and_is_never_repaired(db, bucket): + # A manifest with no backing record at all -- s3_id matches nothing. + backup_manifest.write(_config(), _manifest(_uuid("ghost"))) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is False + assert backup_controller.check_backups(cluster_id=CLUSTER, repair=True) is False + + +def test_a_merged_backups_leftover_manifest_is_not_flagged(db, bucket): + """MERGED/MERGING are excluded: the merge path does not mark manifest + state, so a leftover manifest here is a known gap, not this check's to + report.""" + b = _backup(db, "old", status=Backup.STATUS_MERGED) + backup_manifest.write(_config(), _manifest(b.uuid, s3_id=b.s3_id)) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is True + + +def test_scoping_to_one_cluster_ignores_another_clusters_problems(db, bucket): + b = _backup(db, "leaf", cluster_id=CLUSTER) + backup_manifest.write(_config(), _manifest(b.uuid, s3_id=b.s3_id)) + # No manifest for this one -- a real problem, but in a different cluster. + _backup(db, "other-leaf", cluster_id=OTHER_CLUSTER) + + assert backup_controller.check_backups(cluster_id=CLUSTER) is True + assert backup_controller.check_backups(cluster_id=OTHER_CLUSTER) is False diff --git a/tests/integration/backup/test_delete_backup.py b/tests/integration/backup/test_delete_backup.py index c0cc3b7a5d..9b3dd75d5f 100644 --- a/tests/integration/backup/test_delete_backup.py +++ b/tests/integration/backup/test_delete_backup.py @@ -1,26 +1,42 @@ -"""delete_backup removes one backup record, and refuses what would break a restore, -against real FoundationDB. +"""Deleting (or merging away) one backup, and deleting every backup of a +volume, against real FoundationDB. -Deleting a backup another one chains against makes every descendant -unrestorable, and deleting a member of a group backup that still exists tears -that group backup. Both are refused with an exception rather than reported in a -return value, so a caller cannot forget to look. +A backup with nothing chaining against it is claimed for deletion: its +status moves to DELETING, its manifest is marked DELETING in the bucket, and +a task is queued to remove its S3 objects. A backup something chains against +is instead merged into that successor -- deleting it and merging it are the +same request once a successor exists (``trigger_merge``). Both forms go +through the same chain-wide guard (``DBController.claim_backup_chain_op``), +so a chain already claimed by another merge, delete, or an active restore +refuses a second one. -The database is real. The only boundary patched is the storage node's RPC -client, which the integration tier never reaches a real node for. +The database is real. The bucket is the one boundary faked: the S3 client the +manifest functions use is an in-memory one, so manifest marking runs for +real over it. """ -import uuid as uuid_mod +import io import zlib -from unittest.mock import patch +from uuid import UUID import pytest +from botocore.exceptions import ClientError from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import group_backup +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import ( + BackupManifest, + BackupManifestState, + ManifestDataPlane, + ManifestSource, + ManifestVolume, +) from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup, GroupBackup -from simplyblock_core.models.storage_node import StorageNode -from simplyblock_core.rpc_client import RPCException +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.job_schedule import JobSchedule def _uuid(name: str) -> str: @@ -29,48 +45,97 @@ def _uuid(name: str) -> str: CLUSTER = _uuid("cluster") LVOL = _uuid("lvol") +NODE = _uuid("node") +BUCKET = "simplyblock-backup-cluster" + + +def _config(): + return BackupConfig.model_validate({"bucket_name": BUCKET, "region": "eu-central-1"}) @pytest.fixture def db(): - return DBController() + db = DBController() + c = Cluster() + c.uuid = CLUSTER + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return db + + +class _Bucket: + """An in-memory S3 bucket, behind the manifest calls and the data-object + listing/deletion calls ``backup_manifest.delete_data_objects`` makes.""" + + def __init__(self): + self.objects: dict = {} + + def put_object(self, Bucket, Key, Body, **_): + self.objects[Key] = Body + + def get_object(self, Bucket, Key): + if Key not in self.objects: + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") + return {"Body": io.BytesIO(self.objects[Key])} + + def delete_object(self, Bucket, Key): + self.objects.pop(Key, None) + + def get_paginator(self, _name): + objects = self.objects + + class _Paginator: + def paginate(self, Bucket, Prefix): + yield {"Contents": [{"Key": k} for k in objects if k.startswith(Prefix)]} + return _Paginator() @pytest.fixture -def node(db): - """An online node whose RPC client is patched: the node has no reachable - address, and the S3 delete is the only call made to it.""" - n = StorageNode() - n.uuid = str(uuid_mod.uuid4()) - n.cluster_id = CLUSTER - n.status = StorageNode.STATUS_ONLINE - n.hostname = "delete-backup-host" - n.mgmt_ip = "127.0.0.1" - n.write_to_db(db.kv_store) - with patch.object(StorageNode, "rpc_client") as rpc: - n.rpc = rpc - yield n - - -def _backup(db, node, name, prev="", group_backup_id=""): +def bucket(monkeypatch): + """The backup manifests in the bucket. The manifest functions run for + real over it; only the S3 client is faked.""" + b = _Bucket() + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + return b + + +def _manifest(backup_id: str) -> BackupManifest: + return BackupManifest( + backup_id=UUID(backup_id), + s3_id=zlib.crc32(backup_id.encode()) % 1000, + created_at=100, + completed_at=200, + size=4096, + source=ManifestSource(cluster_id=UUID(CLUSTER), node_id=UUID(NODE)), + volume=ManifestVolume( + lvol_id=UUID(LVOL), lvol_name="vol", snapshot_id=UUID(_uuid("snap")), + snapshot_name="snap", size=4096), + dataplane=ManifestDataPlane(), + ) + + +def _backup(db, bucket, name, prev="", group_backup_id="", status=Backup.STATUS_COMPLETED): b = Backup() b.uuid = _uuid(name) b.cluster_id = CLUSTER b.lvol_id = LVOL - b.node_id = node.uuid + b.node_id = NODE b.s3_id = zlib.crc32(name.encode()) % 1000 b.prev_backup_id = _uuid(prev) if prev else "" b.group_backup_id = group_backup_id - b.status = Backup.STATUS_COMPLETED + b.status = status + b.location = _config().location().model_dump(mode="json") b.write_to_db(db.kv_store) + if status == Backup.STATUS_COMPLETED: + backup_manifest.write(_config(), _manifest(b.uuid)) return b -def _group_backup(db): +def _group_backup(db, status=GroupBackup.STATUS_COMPLETED): gb = GroupBackup() gb.uuid = _uuid("gb") gb.cluster_id = CLUSTER - gb.status = GroupBackup.STATUS_COMPLETED + gb.status = status gb.write_to_db(db.kv_store) return gb @@ -83,32 +148,135 @@ def _exists(db, backup): return False -def test_a_backup_nothing_depends_on_is_deleted(db, node): - leaf = _backup(db, node, "leaf") +def _status(db, backup): + return db.get_backup_by_id(backup.uuid).status + + +def _manifest_state(bucket, backup): + return backup_manifest._parse( + bucket.objects[backup_manifest.manifest_key(UUID(backup.uuid))], + "test").state + + +def _delete_tasks(db, backup_id=None): + tasks = [t for t in db.get_job_tasks_by_function(CLUSTER, JobSchedule.FN_BACKUP_DELETE) + if t.status != JobSchedule.STATUS_DONE and not t.canceled] + if backup_id is None: + return tasks + return [t for t in tasks if backup_id in (t.function_params or {}).get("backup_ids", [])] + + +def _merge_tasks(db, old_backup_id=None): + tasks = [t for t in db.get_job_tasks_by_function(CLUSTER, JobSchedule.FN_BACKUP_MERGE) + if t.status != JobSchedule.STATUS_DONE and not t.canceled] + if old_backup_id is None: + return tasks + return [t for t in tasks if (t.function_params or {}).get("old_backup_id") == old_backup_id] + + +def _restore_task(db, chain_s3_ids): + """A live FN_BACKUP_RESTORE task reading ``chain_s3_ids``, so the restore + index picks it up.""" + task, err = db.create_backup_restore_task_if_chain_clear( + LVOL, CLUSTER, NODE, + function_params={"backup_id": "", "lvol_name": "x", "lvol_id": "", + "chain_ids": chain_s3_ids, "s3_config": None}, + max_retry=1) + assert err is None, err + return task + + +def test_a_leaf_backup_is_claimed_and_queued_for_deletion(db, bucket): + leaf = _backup(db, bucket, "leaf") + assert backup_controller.delete_backup(leaf.uuid) is None - assert not _exists(db, leaf) + assert _exists(db, leaf) + assert _status(db, leaf) == Backup.STATUS_DELETING + assert _manifest_state(bucket, leaf) == BackupManifestState.DELETING + assert _delete_tasks(db, leaf.uuid) -def test_a_backup_another_chains_against_is_refused(db, node): - base = _backup(db, node, "base") - _backup(db, node, "next", prev="base") - with pytest.raises(PreconditionError, match="predecessor"): + +def test_resuming_an_already_deleting_backup_requeues_without_reclaiming(db, bucket): + leaf = _backup(db, bucket, "leaf") + backup_controller.delete_backup(leaf.uuid) + first_tasks = {t.uuid for t in _delete_tasks(db, leaf.uuid)} + + backup_controller.delete_backup(leaf.uuid) + + assert _status(db, leaf) == Backup.STATUS_DELETING + assert {t.uuid for t in _delete_tasks(db, leaf.uuid)} == first_tasks + + +def test_a_backup_with_a_completed_successor_is_merged(db, bucket): + base = _backup(db, bucket, "base") + nxt = _backup(db, bucket, "next", prev="base") + + assert backup_controller.delete_backup(base.uuid) is None + + assert _status(db, base) == Backup.STATUS_MERGING + assert _status(db, nxt) == Backup.STATUS_COMPLETED + assert _merge_tasks(db, base.uuid) + + +def test_a_backup_with_an_incomplete_successor_is_refused(db, bucket): + base = _backup(db, bucket, "base") + _backup(db, bucket, "next", prev="base", status=Backup.STATUS_IN_PROGRESS) + + with pytest.raises(PreconditionError, match="completed"): + backup_controller.delete_backup(base.uuid) + + assert _status(db, base) == Backup.STATUS_COMPLETED + + +def test_a_forked_predecessor_raises_instead_of_merging_into_one_arbitrarily(db, bucket): + """Two backups naming the same predecessor is a chain fork that should + never occur (snapshot chains are linear and backup creation locks the + whole ancestry) -- if one gets into the chain anyway (e.g. a corrupted + import), deleting the predecessor must fail loudly, not silently merge it + into whichever fork happens to be completed and ignore the other, even + one still being uploaded.""" + base = _backup(db, bucket, "base") + _backup(db, bucket, "completed_fork", prev="base") + _backup(db, bucket, "uploading_fork", prev="base", status=Backup.STATUS_IN_PROGRESS) + + with pytest.raises(PreconditionError, match="multiple successors"): backup_controller.delete_backup(base.uuid) - assert _exists(db, base) + assert _status(db, base) == Backup.STATUS_COMPLETED -def test_a_member_of_a_group_backup_that_exists_is_refused(db, node): + +def test_a_member_of_a_group_backup_that_exists_is_refused(db, bucket): gb = _group_backup(db) - member = _backup(db, node, "member", group_backup_id=gb.get_id()) + member = _backup(db, bucket, "member", group_backup_id=gb.get_id()) + with pytest.raises(PreconditionError, match="group backup"): backup_controller.delete_backup(member.uuid) - assert _exists(db, member) + + assert _status(db, member) == Backup.STATUS_COMPLETED -def test_a_member_whose_group_backup_is_gone_is_deleted(db, node): - orphan = _backup(db, node, "orphan", group_backup_id=f"{CLUSTER}/{_uuid('gone')}") +def test_a_member_whose_group_backup_is_gone_is_queued_for_deletion(db, bucket): + orphan = _backup(db, bucket, "orphan", group_backup_id=f"{CLUSTER}/{_uuid('gone')}") + backup_controller.delete_backup(orphan.uuid) - assert not _exists(db, orphan) + + assert _status(db, orphan) == Backup.STATUS_DELETING + + +def test_retire_orphaned_member_backups_raises_on_a_forked_chain(db, bucket): + """retire_orphaned_member_backups indexes a chain by prev_backup_id to + find each orphan's successor; two backups sharing one predecessor must + raise instead of silently keeping whichever the index construction saw + last and losing the other.""" + orphan = _backup(db, bucket, "orphan", group_backup_id=f"{CLUSTER}/{_uuid('gone')}") + _backup(db, bucket, "next_a", prev="orphan") + _backup(db, bucket, "next_b", prev="orphan") + + with pytest.raises(PreconditionError, match="multiple successors"): + group_backup.retire_orphaned_member_backups([LVOL], db=db) + + assert _status(db, orphan) == Backup.STATUS_COMPLETED def test_an_unknown_backup_raises(db): @@ -116,9 +284,181 @@ def test_an_unknown_backup_raises(db): backup_controller.delete_backup(_uuid("nope")) -def test_a_failed_s3_delete_still_deletes_the_record(db, node): - """The S3 delete is best effort: the data plane does not implement it yet.""" - leaf = _backup(db, node, "leaf") - node.rpc.return_value.bdev_lvol_s3_delete.side_effect = RPCException("no such method") +def test_a_chain_already_being_merged_refuses_a_delete_of_another_backup(db, bucket): + """Two leaves of the same volume's chain, one already MERGING -- claiming + the other must see it and refuse.""" + base = _backup(db, bucket, "base") + nxt = _backup(db, bucket, "next", prev="base") + other_leaf = _backup(db, bucket, "other") + ok, reason = backup_controller.trigger_merge(nxt, base) + assert ok, reason + + with pytest.raises(PreconditionError, match="already"): + backup_controller.delete_backup(other_leaf.uuid) + + assert _status(db, other_leaf) == Backup.STATUS_COMPLETED + + +def test_a_backup_claimed_for_delete_refuses_a_concurrent_merge_claim(db, bucket): + """claim_backup_chain_op's own-targets exemption must only cover a target + already sitting in new_status (resuming the identical claim) -- not a + target held in the OTHER status by a genuinely different operation.""" + base = _backup(db, bucket, "base") + _backup(db, bucket, "next", prev="base") + + ok, reason = db.claim_backup_chain_op(LVOL, [base.uuid], Backup.STATUS_DELETING) + assert ok, reason + + ok, reason = db.claim_backup_chain_op(LVOL, [base.uuid], Backup.STATUS_MERGING) + + assert not ok + assert base.uuid in reason + assert _status(db, base) == Backup.STATUS_DELETING + + +def test_a_chain_read_by_an_active_restore_refuses_a_delete(db, bucket): + leaf = _backup(db, bucket, "leaf") + _restore_task(db, [leaf.s3_id]) + + with pytest.raises(PreconditionError, match="restore"): + backup_controller.delete_backup(leaf.uuid) + + assert _status(db, leaf) == Backup.STATUS_COMPLETED + + +def test_a_chain_read_by_an_active_restore_refuses_a_new_restore_task(db, bucket): + """create_backup_restore_task_if_chain_clear itself refuses once a sibling + is DELETING -- the other half of the same guard.""" + leaf = _backup(db, bucket, "leaf") backup_controller.delete_backup(leaf.uuid) - assert not _exists(db, leaf) + + task, err = db.create_backup_restore_task_if_chain_clear( + LVOL, CLUSTER, NODE, + function_params={"backup_id": leaf.uuid, "lvol_name": "x", "lvol_id": "", + "chain_ids": [leaf.s3_id], "s3_config": None}, + max_retry=1) + + assert task is None + assert "deleting" in err + + +def test_delete_backups_refuses_while_a_backup_is_still_being_created(db, bucket): + _backup(db, bucket, "done") + _backup(db, bucket, "uploading", status=Backup.STATUS_IN_PROGRESS) + + ok, err = backup_controller.delete_backups(LVOL) + + assert not ok + assert "still being created" in err + + +def test_delete_backups_refuses_while_a_backup_is_merging(db, bucket): + base = _backup(db, bucket, "base") + nxt = _backup(db, bucket, "next", prev="base") + ok, reason = backup_controller.trigger_merge(nxt, base) + assert ok, reason + + ok, err = backup_controller.delete_backups(LVOL) + + assert not ok + assert "merged" in err + + +def test_delete_backups_refuses_for_a_live_group_member(db, bucket): + gb = _group_backup(db) + _backup(db, bucket, "member", group_backup_id=gb.get_id()) + + ok, err = backup_controller.delete_backups(LVOL) + + assert not ok + assert "group backup" in err + + +def test_delete_backups_claims_completed_backups_and_removes_failed_ones_directly(db, bucket): + completed = _backup(db, bucket, "completed") + failed = _backup(db, bucket, "failed", status=Backup.STATUS_FAILED) + + ok, err = backup_controller.delete_backups(LVOL) + + assert ok, err + assert _status(db, completed) == Backup.STATUS_DELETING + assert _manifest_state(bucket, completed) == BackupManifestState.DELETING + assert _delete_tasks(db, completed.uuid) + assert not _exists(db, failed) + + +def test_delete_backups_is_a_noop_resume_once_everything_is_already_deleting(db, bucket): + _backup(db, bucket, "leaf") + ok, err = backup_controller.delete_backups(LVOL) + assert ok, err + + ok, err = backup_controller.delete_backups(LVOL) + + assert ok, err + + +def _seed_data_objects(bucket, backup): + """Fake data the data plane would have written under this backup's + ``s3_id`` -- a metadata object (``mid=1``) and one data cluster + (``mid=0``), per ``ManifestDataPlane.key_format``.""" + bucket.objects[f"{backup.s3_id}/1/0"] = b"metadata" + bucket.objects[f"{backup.s3_id}/0/0"] = b"cluster-0" + + +class TestDeleteTask: + """``FN_BACKUP_DELETE`` (``tasks_runner_backup._run_delete``): the task + ``backup_controller.delete_backup``/``delete_backups`` queue after + claiming a backup, which removes its S3 objects itself -- there is no + data-plane RPC for this and none is planned. A backup with a successor is + never reached here: that goes through a merge instead + (``trigger_merge``/``_run_merge``), which is where the data plane stays + involved. + """ + + def _task(self, db, backup_id): + return _delete_tasks(db, backup_id)[0] + + def test_deleting_removes_data_objects_manifest_and_record(self, db, bucket): + from simplyblock_core.services import tasks_runner_backup as runner + leaf = _backup(db, bucket, "leaf") + _seed_data_objects(bucket, leaf) + backup_controller.delete_backup(leaf.uuid) + + runner._run_delete(self._task(db, leaf.uuid)) + + assert not any(k.startswith(f"{leaf.s3_id}/") for k in bucket.objects) + assert backup_manifest.manifest_key(UUID(leaf.uuid)) not in bucket.objects + assert not _exists(db, leaf) + + def test_a_backup_no_longer_deleting_is_skipped(self, db, bucket): + """A prior cycle of this same task (or, for a group member, a + different caller) already retired it -- resuming must not re-delete + objects out from under whatever claimed it next.""" + from simplyblock_core.services import tasks_runner_backup as runner + leaf = _backup(db, bucket, "leaf") + backup_controller.delete_backup(leaf.uuid) + task = self._task(db, leaf.uuid) + db.atomic_update(leaf, lambda b: setattr(b, "status", Backup.STATUS_COMPLETED)) + + runner._run_delete(task) + + assert _status(db, leaf) == Backup.STATUS_COMPLETED + + def test_a_bucket_failure_retries_without_touching_the_manifest_or_record( + self, db, bucket, monkeypatch): + from simplyblock_core.services import tasks_runner_backup as runner + from simplyblock_core.services.task_runner_base import TaskRetry + leaf = _backup(db, bucket, "leaf") + _seed_data_objects(bucket, leaf) + backup_controller.delete_backup(leaf.uuid) + + def unreachable(_name): + raise ClientError({"Error": {"Code": "500"}}, "ListObjectsV2") + monkeypatch.setattr(bucket, "get_paginator", unreachable) + + with pytest.raises(TaskRetry): + runner._run_delete(self._task(db, leaf.uuid)) + + assert _status(db, leaf) == Backup.STATUS_DELETING + assert _manifest_state(bucket, leaf) == BackupManifestState.DELETING + assert any(k.startswith(f"{leaf.s3_id}/") for k in bucket.objects) diff --git a/tests/integration/backup/test_group_backup_delete.py b/tests/integration/backup/test_group_backup_delete.py index c0eb99a7b0..c37a3007a6 100644 --- a/tests/integration/backup/test_group_backup_delete.py +++ b/tests/integration/backup/test_group_backup_delete.py @@ -147,6 +147,23 @@ def _two_generations(db, bucket): return gb, base, leaf, successor +def _rogue_backup(db, uuid, lvol_id, prev_backup_id): + """A backup outside any group backup that chains against *prev_backup_id* + -- for building a chain fork: two backups naming the same predecessor.""" + b = Backup() + b.uuid = uuid + b.cluster_id = CLUSTER + b.lvol_id = lvol_id + b.node_id = NODE + b.prev_backup_id = prev_backup_id + b.size = 4096 + b.created_at = 999 + b.status = Backup.STATUS_COMPLETED + b.location = _config().location().model_dump(mode="json") + b.write_to_db(db.kv_store) + return b + + def _status(db, backup): try: return db.get_backup_by_id(backup.get_id()).status @@ -218,7 +235,9 @@ def test_a_leaf_is_deleted_and_a_base_merged_while_the_group_stays(self, db, buc state = group_backup.advance_group_backup_delete(gb.get_id()) assert state == group_backup.DeleteState.PENDING - assert _status(db, leaf) is None + # Claimed, not yet actually gone: leaf's own FN_BACKUP_DELETE task + # removes it once it confirms the S3 objects are gone. + assert _status(db, leaf) == Backup.STATUS_DELETING assert _status(db, base) == Backup.STATUS_MERGING assert db.get_group_backup_by_id(gb.get_id()).status == GroupBackup.STATUS_DELETING assert bucket[GEN_7].state == GroupManifestState.DELETING @@ -230,6 +249,9 @@ def test_the_manifest_and_record_go_only_once_every_member_is_retired(self, db, merged = db.get_backup_by_id(base.get_id()) merged.status = Backup.STATUS_MERGED # the merge task finished merged.write_to_db(db.kv_store) + # leaf's own FN_BACKUP_DELETE task finished too: it confirmed the S3 + # objects were gone and removed the record. + db.get_backup_by_id(leaf.get_id()).remove(db.kv_store) state = group_backup.advance_group_backup_delete(gb.get_id()) @@ -248,6 +270,21 @@ def test_the_next_generation_is_left_whole(self, db, bucket): assert db.get_group_backup_by_id(f"{CLUSTER}/{GEN_8}").status == GroupBackup.STATUS_COMPLETED assert bucket[GEN_8].state == GroupManifestState.COMMITTED + def test_a_forked_base_raises_instead_of_picking_a_successor_arbitrarily(self, db, bucket): + """Two backups naming the same predecessor is a chain fork that should + never occur (snapshot chains are linear and backup creation locks the + whole ancestry) -- if one gets into the chain anyway (e.g. a corrupted + import), retiring the predecessor must fail loudly, not silently merge + it into whichever fork next() happens to see first.""" + gb, base, leaf, successor = _two_generations(db, bucket) + _rogue_backup(db, "0ae09999-0000-4000-8000-000000000099", base.lvol_id, base.uuid) + group_backup.delete_group_backup(gb.get_id()) + + with pytest.raises(PreconditionError, match="multiple successors"): + group_backup.advance_group_backup_delete(gb.get_id()) + + assert _status(db, base) == Backup.STATUS_COMPLETED + def test_a_failed_group_backup_has_no_manifest_to_mark(self, db, bucket): member = _member(db, GEN_7, 7, 1, status=Backup.STATUS_FAILED) gb = _group_backup(db, bucket, GEN_7, 7, [member], status=GroupBackup.STATUS_FAILED) diff --git a/tests/integration/backup/test_legacy.py b/tests/integration/backup/test_legacy.py index a0e68f6883..6dd8bd67f2 100644 --- a/tests/integration/backup/test_legacy.py +++ b/tests/integration/backup/test_legacy.py @@ -708,7 +708,7 @@ def test_success(self, mock_tasks): self.assertEqual(result, "lvol-new") # s3_id integers reach the data plane, not backup UUIDs. - self.assertEqual(mock_tasks.add_backup_restore_task.call_args[0][4], [5]) + self.assertEqual(mock_tasks.add_backup_restore_task.call_args[0][5], [5]) self.assertEqual(self.db.get_lvol_by_id("lvol-new").status, LVol.STATUS_RESTORING) def test_backup_not_found(self): @@ -743,22 +743,33 @@ def test_incomplete_chain_is_refused(self): class TestDeleteBackups(unittest.TestCase): - @patch("simplyblock_core.controllers.backup.controller.backup_events") - @patch("simplyblock_core.models.storage_node.RPCClient") - @patch("simplyblock_core.controllers.backup.controller.db_controller") - def test_success(self, mock_db, MockRPC, mock_events): - b1 = _backup(uuid="b-1") - mock_db.get_backups_by_lvol_id.return_value = [b1] - mock_db.get_storage_node_by_id.return_value = _node() - b1.remove = MagicMock() - MockRPC.return_value.bdev_lvol_s3_delete.return_value = True + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") + def test_success(self, mock_boto3_client): + # delete_backups claims the chain and marks the manifest DELETING + # against the real database and a real (faked) S3 client -- it no + # longer removes the record synchronously, so there is nothing left + # here to mock away the way the old, synchronous delete_backups did. + from botocore.exceptions import ClientError + mock_boto3_client.return_value.get_object.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey"}}, "GetObject") + + db = DBController() + cluster = _cluster() + cluster.write_to_db(db.kv_store) + # A real UUID, not the short "b-1" placeholder used elsewhere in this + # file: the manifest path now parses backup.uuid with UUID(...) + # (manifest_key, build_manifest's backup_id), which a short string + # like "b-1" is not shaped like. + backup_uuid = "00000000-0000-4000-8000-000000000001" + b1 = _backup(uuid=backup_uuid) + b1.location = _backup_config().location().model_dump(mode="json") + b1.write_to_db(db.kv_store) from simplyblock_core.controllers.backup.controller import delete_backups success, error = delete_backups("lvol-1") - self.assertTrue(success) - self.assertIsNone(error) - b1.remove.assert_called_once() + self.assertTrue(success, error) + self.assertEqual(db.get_backup_by_id(backup_uuid).status, Backup.STATUS_DELETING) @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_no_backups(self, mock_db): @@ -984,17 +995,24 @@ def test_no_policy(self, mock_db, mock_tasks): mock_tasks.add_backup_merge_task.assert_not_called() - @patch.object(Backup, 'write_to_db') @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") - def test_version_limit_exceeded(self, mock_db, mock_tasks, _mock_write): + def test_version_limit_exceeded(self, mock_db, mock_tasks): policy = _policy(max_versions=2, max_age_seconds=0) mock_db.get_policy_for_lvol.return_value = policy + # trigger_merge's own chain guard (claim_backup_chain_op) reads the + # REAL database, not policy.py's mocked one above -- see the module + # docstring on why this file is legacy and tests/AGENTS.md on why a + # patched DB only covers the module it names. The backups the guard + # has to find therefore have to be real. + db = DBController() now = int(time.time()) b1 = _backup(uuid="b1", created_at=now - 300) b2 = _backup(uuid="b2", created_at=now - 200) b3 = _backup(uuid="b3", created_at=now - 100) + for b in (b1, b2, b3): + b.write_to_db(db.kv_store) mock_db.get_backups_by_lvol_id.return_value = [b1, b2, b3] from simplyblock_core.controllers.backup.policy import evaluate_policy @@ -1020,17 +1038,19 @@ def test_under_version_limit(self, mock_db, mock_tasks): mock_tasks.add_backup_merge_task.assert_not_called() - @patch.object(Backup, 'write_to_db') @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") - def test_age_limit_exceeded(self, mock_db, mock_tasks, _mock_write): + def test_age_limit_exceeded(self, mock_db, mock_tasks): policy = _policy(max_versions=0) policy.max_age_seconds = 3600 # 1 hour mock_db.get_policy_for_lvol.return_value = policy + db = DBController() now = int(time.time()) b1 = _backup(uuid="b1", created_at=now - 7200) # 2 hours old b2 = _backup(uuid="b2", created_at=now - 100) + for b in (b1, b2): + b.write_to_db(db.kv_store) mock_db.get_backups_by_lvol_id.return_value = [b1, b2] from simplyblock_core.controllers.backup.policy import evaluate_policy @@ -1039,18 +1059,20 @@ def test_age_limit_exceeded(self, mock_db, mock_tasks, _mock_write): mock_tasks.add_backup_merge_task.assert_called_once() - @patch.object(Backup, 'write_to_db') @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") - def test_both_conditions_required(self, mock_db, mock_tasks, _mock_write): + def test_both_conditions_required(self, mock_db, mock_tasks): """When both versions and age are set, either limit can trigger a merge.""" policy = _policy(max_versions=3) policy.max_age_seconds = 3600 mock_db.get_policy_for_lvol.return_value = policy + db = DBController() now = int(time.time()) # 4 backups (exceeds version limit of 3) but all recent (doesn't exceed age) backups = [_backup(uuid=f"b{i}", created_at=now - (i * 60)) for i in range(4)] + for b in backups: + b.write_to_db(db.kv_store) mock_db.get_backups_by_lvol_id.return_value = backups from simplyblock_core.controllers.backup.policy import evaluate_policy @@ -1059,19 +1081,21 @@ def test_both_conditions_required(self, mock_db, mock_tasks, _mock_write): mock_tasks.add_backup_merge_task.assert_called_once() - @patch.object(Backup, 'write_to_db') @patch("simplyblock_core.controllers.backup.controller.tasks_controller") @patch("simplyblock_core.controllers.backup.policy.db_controller") - def test_both_conditions_met(self, mock_db, mock_tasks, _mock_write): + def test_both_conditions_met(self, mock_db, mock_tasks): """When both limits set and both exceeded, merge triggers.""" policy = _policy(max_versions=2) policy.max_age_seconds = 3600 mock_db.get_policy_for_lvol.return_value = policy + db = DBController() now = int(time.time()) b1 = _backup(uuid="b1", created_at=now - 7200) b2 = _backup(uuid="b2", created_at=now - 200) b3 = _backup(uuid="b3", created_at=now - 100) + for b in (b1, b2, b3): + b.write_to_db(db.kv_store) mock_db.get_backups_by_lvol_id.return_value = [b1, b2, b3] from simplyblock_core.controllers.backup.policy import evaluate_policy @@ -1179,9 +1203,14 @@ def test_bdev_lvol_s3_recovery_exists(self): from simplyblock_core.rpc_client import RPCClient self.assertTrue(hasattr(RPCClient, 'bdev_lvol_s3_recovery')) - def test_bdev_lvol_s3_delete_exists(self): + def test_bdev_lvol_s3_delete_is_gone(self): + """There is no data-plane RPC for deleting a backup's S3 objects, and + none is planned -- the control plane removes them itself + (backup_manifest.delete_data_objects), the same way it removes a + manifest. A backup that still has a successor goes through + bdev_lvol_s3_merge instead.""" from simplyblock_core.rpc_client import RPCClient - self.assertTrue(hasattr(RPCClient, 'bdev_lvol_s3_delete')) + self.assertFalse(hasattr(RPCClient, 'bdev_lvol_s3_delete')) # =========================================================================== @@ -1295,14 +1324,21 @@ class TestTriggerMerge(unittest.TestCase): @patch("simplyblock_core.controllers.backup.controller.tasks_controller") def test_trigger_merge_marks_old_as_merging(self, mock_tasks): from simplyblock_core.controllers.backup.controller import trigger_merge + # trigger_merge's chain guard (claim_backup_chain_op) flips the status + # of the record it finds in the REAL database, inside its own + # transaction -- never the caller's in-memory object -- so both + # backups have to be real, and the result is read back through the + # database too. + db = DBController() keep = _backup(uuid="keep") old = _backup(uuid="old") - old.write_to_db = MagicMock() + keep.write_to_db(db.kv_store) + old.write_to_db(db.kv_store) - trigger_merge(keep, old) + ok, reason = trigger_merge(keep, old) - self.assertEqual(old.status, Backup.STATUS_MERGING) - old.write_to_db.assert_called_once() + self.assertTrue(ok, reason) + self.assertEqual(db.get_backup_by_id("old").status, Backup.STATUS_MERGING) mock_tasks.add_backup_merge_task.assert_called_once() @patch("simplyblock_core.controllers.backup.controller.tasks_controller") diff --git a/tests/unit/backup/test_chain.py b/tests/unit/backup/test_chain.py index 9f9fbdbecd..ac19d421a4 100644 --- a/tests/unit/backup/test_chain.py +++ b/tests/unit/backup/test_chain.py @@ -14,7 +14,11 @@ import pytest from simplyblock_core import constants -from simplyblock_core.controllers.backup.chain import BackupChain, location_of +from simplyblock_core.controllers.backup.chain import ( + BackupChain, + SuccessorIndex, + location_of, +) from simplyblock_core.controllers.backup.manifest import ( BackupManifest, FDBKeyDescriptor, @@ -273,6 +277,42 @@ def test_the_wording_names_what_was_refused(self): what="This snapshot chain") +class TestSuccessorIndex: + + def test_a_backup_with_no_dependent_has_no_successor(self): + index = SuccessorIndex.of_lvol(_line_of_records(2)) + assert index.successor_of(str(_id("b-1"))) is None + + def test_finds_the_backup_naming_it_as_prev(self): + line = _line_of_records(3) + index = SuccessorIndex.of_lvol(line) + successor_of_0 = index.successor_of(str(_id("b-0"))) + successor_of_1 = index.successor_of(str(_id("b-1"))) + assert successor_of_0 is not None and successor_of_0.uuid == str(_id("b-1")) + assert successor_of_1 is not None and successor_of_1.uuid == str(_id("b-2")) + + def test_order_of_the_population_does_not_matter(self): + line = _line_of_records(3) + index = SuccessorIndex.of_lvol(list(reversed(line))) + successor = index.successor_of(str(_id("b-0"))) + assert successor is not None and successor.uuid == str(_id("b-1")) + + def test_a_backup_naming_no_predecessor_is_ignored(self): + """A full backup's empty prev_backup_id must not collide with another + full backup's, as if both were "the successor of nothing".""" + a = _record("full-a") + b = _record("full-b") + index = SuccessorIndex.of_lvol([a, b]) + assert index.successor_of("") is None + + def test_two_backups_naming_the_same_predecessor_is_a_fork(self): + base = _record("base") + fork_a = _record("fork-a", prev="base") + fork_b = _record("fork-b", prev="base") + with pytest.raises(PreconditionError, match="multiple successors"): + SuccessorIndex.of_lvol([base, fork_a, fork_b]) + + class TestS3Ids: def test_newest_first(self): diff --git a/tests/unit/backup/test_manifest_delete_state.py b/tests/unit/backup/test_manifest_delete_state.py new file mode 100644 index 0000000000..7a3cdc762f --- /dev/null +++ b/tests/unit/backup/test_manifest_delete_state.py @@ -0,0 +1,139 @@ +"""Marking a single-backup manifest DELETING/ACTIVE, ahead of actually +removing anything. + +A delete marks a backup's manifest DELETING before any of its S3 objects are +touched, and reverts it to ACTIVE if the delete does not finish -- the +single-backup half of the metadata-before-data ordering the group manifest's +``GroupManifestState`` already keeps. Both functions are read-modify-write +over the bucket alone; no database is involved. + +The S3 client is the one boundary faked, as an in-memory bucket, following +the same pattern as ``test_group_manifest_delete.py``. +""" +import io +import zlib +from uuid import UUID + +import pytest +from botocore.exceptions import ClientError + +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import ( + BackupManifest, + BackupManifestState, + ManifestDataPlane, + ManifestError, + ManifestSource, + ManifestVolume, +) +from simplyblock_core.models.backup_config import BackupConfig + + +def _id(name: str) -> UUID: + return UUID(f"{zlib.crc32(name.encode()):08x}-0000-4000-8000-000000000000") + + +BACKUP = _id("b-1") +CONFIG = BackupConfig.model_validate({"bucket_name": "backups", "region": "eu-central-1"}) +KEY = backup_manifest.manifest_key(BACKUP) + + +class _Bucket: + """The three object calls mark_deleting/mark_active make.""" + + def __init__(self, fail=False): + self.objects: dict = {} + self.fail = fail + + def _check(self, op): + if self.fail: + raise ClientError({"Error": {"Code": "InternalError"}}, op) + + def put_object(self, Bucket, Key, Body, **_): + self._check("PutObject") + self.objects[Key] = Body + + def get_object(self, Bucket, Key): + self._check("GetObject") + if Key not in self.objects: + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") + return {"Body": io.BytesIO(self.objects[Key])} + + def delete_object(self, Bucket, Key): + self._check("DeleteObject") + self.objects.pop(Key, None) + + +@pytest.fixture +def bucket(monkeypatch): + b = _Bucket() + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + return b + + +def _store(state=BackupManifestState.ACTIVE): + backup_manifest.write(CONFIG, BackupManifest( + backup_id=BACKUP, + s3_id=1, + created_at=100, + completed_at=200, + size=4096, + source=ManifestSource(cluster_id=_id("cluster"), node_id=_id("node")), + volume=ManifestVolume( + lvol_id=_id("lvol"), lvol_name="vol", snapshot_id=_id("snap"), + snapshot_name="snap", size=4096), + dataplane=ManifestDataPlane(), + state=state, + )) + + +def test_mark_deleting_rewrites_an_active_manifest(bucket): + _store(BackupManifestState.ACTIVE) + + backup_manifest.mark_deleting(CONFIG, BACKUP) + + assert backup_manifest.read(CONFIG, BACKUP).state == BackupManifestState.DELETING + + +def test_mark_deleting_is_a_noop_if_already_deleting(bucket): + _store(BackupManifestState.DELETING) + + backup_manifest.mark_deleting(CONFIG, BACKUP) + + assert backup_manifest.read(CONFIG, BACKUP).state == BackupManifestState.DELETING + + +def test_mark_deleting_is_a_noop_if_the_manifest_is_already_gone(bucket): + backup_manifest.mark_deleting(CONFIG, BACKUP) + + assert bucket.objects == {} + + +def test_mark_active_reverts_a_deleting_manifest(bucket): + _store(BackupManifestState.DELETING) + + backup_manifest.mark_active(CONFIG, BACKUP) + + assert backup_manifest.read(CONFIG, BACKUP).state == BackupManifestState.ACTIVE + + +def test_mark_active_is_a_noop_if_already_active(bucket): + _store(BackupManifestState.ACTIVE) + + backup_manifest.mark_active(CONFIG, BACKUP) + + assert backup_manifest.read(CONFIG, BACKUP).state == BackupManifestState.ACTIVE + + +def test_mark_active_is_a_noop_if_the_manifest_is_already_gone(bucket): + backup_manifest.mark_active(CONFIG, BACKUP) + + assert bucket.objects == {} + + +def test_a_bucket_that_cannot_be_read_is_a_manifest_error(monkeypatch): + b = _Bucket(fail=True) + monkeypatch.setattr(backup_manifest, "s3_client", lambda config: b) + + with pytest.raises(ManifestError): + backup_manifest.mark_deleting(CONFIG, BACKUP) diff --git a/tests/unit/backup/test_restore_node_selection.py b/tests/unit/backup/test_restore_node_selection.py index c7d8298186..bb96c04fb6 100644 --- a/tests/unit/backup/test_restore_node_selection.py +++ b/tests/unit/backup/test_restore_node_selection.py @@ -116,7 +116,7 @@ def test_restore_task_targets_the_node_the_volume_landed_on(self, db, add_lvol_h _restore() - cluster_id, node_id, *_ = tasks.add_backup_restore_task.call_args.args + source_lvol_id, cluster_id, node_id, *_ = tasks.add_backup_restore_task.call_args.args assert cluster_id == TARGET_CLUSTER assert node_id == "target-node" @@ -201,10 +201,10 @@ def test_volume_creation_failure_is_a_runtime_error(self, db, add_lvol_ha): with pytest.raises(RuntimeError, match="Failed to create restore volume"): _restore() - def test_task_creation_failure_is_a_runtime_error(self, db, add_lvol_ha, tasks): + def test_task_creation_failure_is_a_precondition_error(self, db, add_lvol_ha, tasks): tasks.add_backup_restore_task.return_value = False - with pytest.raises(RuntimeError, match="Failed to create restore task"): + with pytest.raises(PreconditionError, match="Failed to create restore task"): _restore() diff --git a/tests/unit/test_replication_incident_0929.py b/tests/unit/test_replication_incident_0929.py index 0c54507030..b038a6ad68 100644 --- a/tests/unit/test_replication_incident_0929.py +++ b/tests/unit/test_replication_incident_0929.py @@ -286,8 +286,8 @@ def test_backup_family_is_owned(self): self.assertEqual( set(tasks_runner_backup.SPEC.function_names), {JobSchedule.FN_BACKUP, JobSchedule.FN_BACKUP_RESTORE, - JobSchedule.FN_BACKUP_MERGE, JobSchedule.FN_GROUP_BACKUP_COMMIT, - JobSchedule.FN_GROUP_BACKUP_DELETE}, + JobSchedule.FN_BACKUP_MERGE, JobSchedule.FN_BACKUP_DELETE, + JobSchedule.FN_GROUP_BACKUP_COMMIT, JobSchedule.FN_GROUP_BACKUP_DELETE}, )