Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions simplyblock_cli/cli-reference.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1763,6 +1763,25 @@ commands:
dest: repair
type: bool
action: store_true
- name: check-backups
help: "Verifies every backup's FDB record against its S3 manifest."
usage: >
Checks, in both directions, that every COMPLETED or DELETING backup has a manifest whose state
and s3_id/prev_backup_id agree with the record, and that every ACTIVE manifest in the bucket
names a record that still exists. Read-only unless `--repair` is given, which rewrites a
missing or disagreeing manifest from the record (today's source of truth). An orphaned
manifest - one naming no known record - is never auto-repaired: it may be a disaster-recovery
backup nothing has imported yet. Deployment-wide unless --cluster-id is given.
arguments:
- name: "--cluster-id"
help: "Limit the check to one cluster. Omit to check every cluster."
dest: cluster_id
type: str
- name: "--repair"
help: "Rewrite manifests that are missing or disagree with their record, instead of only reporting them."
dest: repair
type: bool
action: store_true
- name: graceful-shutdown
help: "Initiates a graceful shutdown of a cluster's storage nodes."
arguments:
Expand Down Expand Up @@ -3364,6 +3383,13 @@ commands:
help: "The logical volume id."
dest: lvol_id
type: str
- name: delete-backup
help: "Delete one backup. A backup nothing chains against is deleted; one something chains against is merged into that successor instead."
arguments:
- name: "backup_id"
help: "The backup id."
dest: backup_id
type: str
- name: restore
help: "Restore a backup to a new logical volume."
arguments:
Expand Down
15 changes: 15 additions & 0 deletions simplyblock_cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,7 @@ def init_cluster(self):
self.init_cluster__build_indices(subparser)
self.init_cluster__index_state(subparser)
self.init_cluster__check_indices(subparser)
self.init_cluster__check_backups(subparser)
self.init_cluster__graceful_shutdown(subparser)
self.init_cluster__restart(subparser)
self.init_cluster__graceful_startup(subparser)
Expand Down Expand Up @@ -645,6 +646,11 @@ def init_cluster__check_indices(self, subparser):
subcommand = self.add_sub_command(subparser, 'check-indices', 'Verifies the database\'s secondary indices.')
subcommand.add_argument('--repair', help='Write back the entries that can be derived from the records, and clear orphaned ones, instead of only reporting them.', dest='repair', action='store_true')

def init_cluster__check_backups(self, subparser):
subcommand = self.add_sub_command(subparser, 'check-backups', 'Verifies every backup\'s FDB record against its S3 manifest.')
subcommand.add_argument('--cluster-id', help='Limit the check to one cluster. Omit to check every cluster.', type=str, dest='cluster_id')
subcommand.add_argument('--repair', help='Rewrite manifests that are missing or disagree with their record, instead of only reporting them.', dest='repair', action='store_true')

def init_cluster__graceful_shutdown(self, subparser):
subcommand = self.add_sub_command(subparser, 'graceful-shutdown', 'Initiates a graceful shutdown of a cluster\'s storage nodes.')
subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list
Expand Down Expand Up @@ -1310,6 +1316,7 @@ def init_backup(self):
subparser = self.add_command('backup', 'Backup Commands')
self.init_backup__list(subparser)
self.init_backup__delete(subparser)
self.init_backup__delete_backup(subparser)
self.init_backup__restore(subparser)
self.init_backup__export(subparser)
self.init_backup__discover(subparser)
Expand All @@ -1330,6 +1337,10 @@ def init_backup__delete(self, subparser):
subcommand = self.add_sub_command(subparser, 'delete', 'Delete all backups for a logical volume.')
subcommand.add_argument('lvol_id', help='The logical volume id.', type=str)

def init_backup__delete_backup(self, subparser):
subcommand = self.add_sub_command(subparser, 'delete-backup', 'Delete one backup. A backup nothing chains against is deleted; one something chains against is merged into that successor instead.')
subcommand.add_argument('backup_id', help='The backup id.', type=str)

def init_backup__restore(self, subparser):
subcommand = self.add_sub_command(subparser, 'restore', 'Restore a backup to a new logical volume.')
subcommand.add_argument('backup_id', help='The volume backup id.', type=str)
Expand Down Expand Up @@ -1760,6 +1771,8 @@ def run(self):
ret = self.cluster__index_state(sub_command, args)
elif sub_command in ['check-indices']:
ret = self.cluster__check_indices(sub_command, args)
elif sub_command in ['check-backups']:
ret = self.cluster__check_backups(sub_command, args)
elif sub_command in ['graceful-shutdown']:
ret = self.cluster__graceful_shutdown(sub_command, args)
elif sub_command in ['restart']:
Expand Down Expand Up @@ -2017,6 +2030,8 @@ def run(self):
ret = self.backup__list(sub_command, args)
elif sub_command in ['delete']:
ret = self.backup__delete(sub_command, args)
elif sub_command in ['delete-backup']:
ret = self.backup__delete_backup(sub_command, args)
elif sub_command in ['restore']:
ret = self.backup__restore(sub_command, args)
elif sub_command in ['export']:
Expand Down
16 changes: 14 additions & 2 deletions simplyblock_cli/clibase.py
Original file line number Diff line number Diff line change
Expand Up @@ -767,6 +767,10 @@ def cluster__index_state(self, sub_command, args):
def cluster__check_indices(self, sub_command, args):
return cluster_ops.check_indices(repair=args.repair)

def cluster__check_backups(self, sub_command, args):
return cluster_ops.check_backups(
cluster_id=getattr(args, 'cluster_id', None), repair=args.repair)

def cluster__graceful_shutdown(self, sub_command, args):
cluster_ops.cluster_grace_shutdown(args.cluster_id)
return True
Expand Down Expand Up @@ -1773,8 +1777,16 @@ def backup__delete(self, sub_command, args):
if error:
print(f"Error: {error}")
return False
print("Backups deleted")
return True
return f"Backups of volume {args.lvol_id} are being deleted"

def backup__delete_backup(self, sub_command, args):
try:
backup_controller.delete_backup(args.backup_id)
except KeyError:
return f"Delete failed: backup {args.backup_id} not found"
except PreconditionError as e:
return f"Delete failed: {e}"
return f"Backup {args.backup_id} is being deleted"

def backup__restore(self, sub_command, args):
try:
Expand Down
2 changes: 1 addition & 1 deletion simplyblock_core/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Core business logic, data models, and background services for the Simplyblock co

## Package Structure

- `controllers/` — Business logic per resource domain (lvol, snapshot, backup, device, migration, pool, health, tasks, qos). Each `*_events.py` defines event types for its domain.
- `controllers/` — Business logic per resource domain (lvol, snapshot, backup, device, migration, pool, health, tasks, qos). Each `*_events.py` defines event types for its domain. For the backup domain, `controllers/backup/chain.py` is the canonical home for backup-chain invariants — coherence (`BackupChain`, restore-side) and fork detection (`SuccessorIndex`, delete/retire-side). Check there before hand-rolling a chain walk or a `prev_backup_id` lookup at a new call site.
- `models/` — Data models inheriting from `BaseModel` (see below), plus the two stdlib-only
modules that define the keyspaces those records live in: `watches.py` (the version index
watchers wake on) and `indices.py` (the declared secondary indices). Both are leaves — the
Expand Down
11 changes: 11 additions & 0 deletions simplyblock_core/cluster_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
tasks_controller,
tcp_ports_events,
)
from simplyblock_core.controllers.backup import controller as backup_controller
from simplyblock_core.controllers.backup import device as backup_device
from simplyblock_core.controllers.cluster_expansion.planner import (
RemoteTripletPlacementError,
Expand Down Expand Up @@ -4323,6 +4324,16 @@ def check_indices(repair=False) -> bool:
return findings['unresolved'] == 0


def check_backups(cluster_id=None, repair=False) -> bool:
"""Verify every backup's FDB record agrees with its S3 manifest, in both
directions, and report the drift. The backup analog of
:func:`check_indices`.

Deployment-wide when ``cluster_id`` is omitted.
"""
return backup_controller.check_backups(cluster_id=cluster_id, repair=repair)


def cluster_grace_startup(cl_id, clear_data=False, spdk_image=None) -> None:
get_cluster = db_controller.get_cluster_by_id(cl_id) # ensure exists

Expand Down
47 changes: 47 additions & 0 deletions simplyblock_core/controllers/backup/chain.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,3 +324,50 @@ def s3_ids_newest_first(self) -> list[int]:
data has to win and older backups fill whatever gaps remain.
"""
return [link.s3_id for link in reversed(self.links)]


@dataclass(frozen=True)
class SuccessorIndex:
"""Which backup, if any, is a delta against each backup of one lvol.

The complement of :class:`BackupChain`: that walks backward from a trusted
head; this answers the forward question delete and retire need -- "does
anything chain against this backup" -- over a whole lvol's backups at once,
with no head and no restorability or coherence requirement, since nothing
here is being restored.

A fork -- two backups naming the same ``prev_backup_id`` -- can only reach
storage through backup import, which checks each manifest's own ancestry
but never checks two manifests, or a manifest against an existing record,
against each other. :meth:`of_lvol` is where that defect becomes visible:
it refuses to guess which of the two is "the" successor.
"""

_by_prev: Mapping[str, Backup]

@classmethod
def of_lvol(cls, backups: Iterable[Backup]) -> "SuccessorIndex":
"""The successor of every backup among *backups*, which should be
every ``Backup`` of one lvol -- a successor outside that population
would not be found.

Raises:
PreconditionError: two backups name the same ``prev_backup_id``,
i.e. the chain has forked.
"""
by_prev: dict[str, Backup] = {}
for backup in backups:
if not backup.prev_backup_id:
continue
if (existing := by_prev.get(backup.prev_backup_id)) is not None:
raise PreconditionError(
f"backup {backup.prev_backup_id} has multiple successors: "
f"{existing.uuid} and {backup.uuid}")
by_prev[backup.prev_backup_id] = backup

return cls(by_prev)

def successor_of(self, backup_id: str) -> Backup | None:
"""The backup that is a delta against *backup_id*, or ``None`` if
nothing in this index chains against it."""
return self._by_prev.get(backup_id)
Loading
Loading