diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 971c831f28..82355e7e7e 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -711,6 +711,41 @@ def get_config_var(name, default=None): # released, the volume replicates on its ordinary cadence, and the endgame is # retried after this long (the abandon burns one attempt of the task's budget). REPL_CUTOVER_ENDGAME_BACKOFF_SEC = 300 +# --- a migration must not fill its source cluster (bed bf29, 2026-11-20) --- +# Five endgames of one source cluster at once, under sustained overwrite, took +# a shrink snapshot every 15-30 s each; every one pinned what the client +# overwrote after it, and their deletions fell behind: 41% -> 99% in 16 min. +# See simplyblock_core/migration_space.py. These hold back only the +# migration's own snapshots -- never client IO or client volume creation. +# +# Cutover endgames (shrink-round phase) running at once per SOURCE cluster. +# The others replicate on their ordinary cadence while they wait for a slot. +REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER = 1 +# An endgame takes its next round snapshot only once none of the volume's +# snapshots is still in deletion -- for at most this long per wait: a deletion +# that is stuck (no leader, a clone in deletion) must not stall the endgame +# into its shrink deadline on its own. The free-space check still applies. +REPL_ENDGAME_DELETION_WAIT_MAX_SEC = 300 +# A migration snapshot needs this many times the bytes written to the volume +# since its last snapshot free on the source (the snapshot pins them, and the +# client overwrites about as much again while the round transfers) ... +REPL_MIGRATION_HEADROOM_FACTOR = 2 +# ... and never less than this share of the cluster (other volumes' writes, +# and the capacity record being up to a collector interval old). +REPL_MIGRATION_MIN_FREE_PCT = 5 +# A cluster capacity record older than this is no reading (the collector +# writes one every DEV_STAT_COLLECTOR_INTERVAL_SEC). +REPL_MIGRATION_CAPACITY_STALE_SEC = 120 +# Without a usable capacity reading a migration snapshot waits this long, then +# goes ahead without the check (logged): a broken stats collector must not +# stop migrations for good. +REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC = 120 +# A cutover endgame whose target node is down before its clone exists cannot +# progress, and holds its volume out of ordinary replication while it waits +# (bed bf29 R3b: RPO 1280 s). After this long it is given up -- no attempt +# burned, no back-off -- and the volume replicates on its cadence until the +# target is back. +REPL_CUTOVER_TARGET_DOWN_GRACE_SEC = 60 # --- cutover source throttle ---------------------------------------------- # A round converges to overhead / (1 - W/T): with the application writing at # W and the delta transferring at T, no number of rounds gets under the @@ -896,6 +931,14 @@ def get_config_var(name, default=None): # leader within about a minute of its primary going down (crashed or shut # down); a wait beyond this means no member could be proven safe to lead. REPL_TARGET_LEADERLESS_ALERT_SEC = 180 +# A transfer that has waited this long for a target lvstore whose members are +# ALL online asks the control plane's guarded leaderless recovery +# (storage_node_ops.restore_healthy_lvs_leadership) to give it a leader, then +# at most every REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC. A primary SPDK that +# dropped the leadership itself takes it back only on IO, and a target gets +# none while replication waits (bf32 2026-11-21: LVS_7/LVS_10, 76 min). +REPL_TARGET_LEADERLESS_RECOVER_SEC = 60 +REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC = 120 # Pass interval of the snapshot replication runner while any transfer or # merge is in flight; TASK_EXEC_INTERVAL_SEC when idle. Not sub-second: each # pass reads the task table (see test_transfer_completion_latency). @@ -911,6 +954,13 @@ def get_config_var(name, default=None): REPL_RUNNER_WORKERS = int(os.getenv("REPL_RUNNER_WORKERS", "8")) # Pause before a task that returned a failure is dispatched again. REPL_RUNNER_RETRY_PAUSE_SEC = 3.0 +# An in_creation landing volume (REP_*) whose create is not running in this +# runner and that no other live task of its snapshot may be creating is taken +# for a dead attempt's leftover and deleted only after this long. Landing +# creates take up to ~50 s on a loaded target (bed bf13); bed bf32 (2026-11-21) +# deleted one 2 s into its create, which left a stale registration on the +# secondaries. +REPL_LANDING_ABANDON_SEC = int(os.getenv("REPL_LANDING_ABANDON_SEC", "600")) # Requests a replication transfer keeps in flight. A full transfer sends one # whole cluster per request (16 x 2 MiB = 32 MiB in flight); a delta # (allow_partial) sends at most 64 KiB per request, so it needs a deeper window diff --git a/simplyblock_core/controllers/lvol_events.py b/simplyblock_core/controllers/lvol_events.py index 3b0459661f..e9f572ee7f 100644 --- a/simplyblock_core/controllers/lvol_events.py +++ b/simplyblock_core/controllers/lvol_events.py @@ -199,3 +199,26 @@ def replication_target_leaderless(lvol, lvs_name, waited_s, why, caused_by=ec.CA f"{lvs_name} has had no leader for {int(waited_s)}s ({why})"), node_id=lvol.get_id(), event_level=EventObj.LEVEL_ERROR) + + +def migration_space_low(lvol, reason, caused_by=ec.CAUSED_BY_MONITOR): + """A migration snapshot of *lvol* waits for free space on its source + cluster (simplyblock_core/migration_space.py). Warning level: client IO + is not affected, only the migration's own snapshots wait. Raised once per + wait, when it starts.""" + from simplyblock_core.models.events import EventObj + try: + cluster_id = DBController().get_pool_by_id(lvol.pool_uuid).cluster_id + except Exception as e: + logger.error(f"No cluster could be resolved for lvol {lvol.get_id()}; event not logged: {e}") + return + ec.log_event_cluster( + cluster_id=cluster_id, + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_REPLICATION_STALLED, + db_object=lvol, + caused_by=caused_by, + message=(f"Migration of LVol {lvol.get_id()} is waiting for free space on its " + f"source cluster: {reason}"), + node_id=lvol.get_id(), + event_level=EventObj.LEVEL_WARN) diff --git a/simplyblock_core/controllers/ns_host_guard.py b/simplyblock_core/controllers/ns_host_guard.py index 78f71db259..e7a0981585 100644 --- a/simplyblock_core/controllers/ns_host_guard.py +++ b/simplyblock_core/controllers/ns_host_guard.py @@ -62,8 +62,22 @@ def connected_hosts(nqn: str, node_ids: Iterable[str], db=None) -> set | None: def stranded_hosts(nqn: str, removing: Iterable[str], keeping: Iterable[str], db=None) -> set | None: - """Hosts connected to *nqn* on the *removing* nodes but on none of the - *keeping* nodes; None when that could not be established.""" + """Hosts connected to *nqn* on the *removing* nodes that have no controller + on the *keeping* nodes; None when that could not be established. + + A host NQN does not identify a machine: one Kubernetes node connects with + the kernel default (``/etc/nvme/hostnqn``) on one path and with the + simplyblock form derived from its node UID on another (bed bf28, + 2026-11-19: every client reached the source as + ``nqn.2014-08.io.simplyblock:uuid:`` and the early-prepared + target as ``nqn.2014-08.org.nvmexpress:uuid:``, with different + host ids), and nothing records which forms belong together. So hosts are + matched by name where they can be, and every unmatched host on the keeping + side is taken to be one unmatched host of the removing side under its + other name. Hosts are stranded only when the removing side has more + unmatched hosts than the keeping side can account for; all of the removing + side's unmatched hosts are then reported, since which of them is the + stranded one cannot be told.""" gone = connected_hosts(nqn, removing, db=db) if gone is None: return None @@ -72,7 +86,10 @@ def stranded_hosts(nqn: str, removing: Iterable[str], keeping: Iterable[str], stay = connected_hosts(nqn, keeping, db=db) if stay is None: return None - return gone - stay + unmatched_gone, unmatched_stay = gone - stay, stay - gone + if len(unmatched_gone) <= len(unmatched_stay): + return set() + return unmatched_gone def removal_blocked(nqn: str, removing: Iterable[str], keeping: Iterable[str], diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index e9c7dad063..d37ad3e9a0 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -421,6 +421,16 @@ def _add_task(function_name, cluster_id, node_id, device_id, task_obj.function_params = function_params task_obj.max_retry = max_retry task_obj.status = JobSchedule.STATUS_NEW + if function_name == JobSchedule.FN_SNAPSHOT_REPLICATION: + # The scan above is a check-then-write: two replication workers + # enqueueing the same chain ancestor 100 ms apart both passed it and + # wrote two tasks for one snapshot (bed bf32 2026-11-21). The write + # itself is the atomic claim. + active_id, created = db.ensure_snapshot_replication_task(task_obj) + if not created: + logger.info(f"Task found, skip adding new task: {active_id}") + return False + return task_obj.uuid task_obj.write_to_db(db.kv_store) if send_to_cluster_log: tasks_events.task_create(task_obj) diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index d8a96c85b2..d70ee0c112 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -923,6 +923,46 @@ def ensure_sync_resync_task(self, task: JobSchedule, lvs_name: str) -> tuple[str return fdb.transactional(DBController._ensure_sync_resync_task_tx)( self, self.kv_store, task, lvs_name, JobSchedule.active_indexes(self.kv_store)) + # repl_snap_task// -> db key of the snapshot's + # replication task. Read and written in the transaction that creates the + # task, so two creators of the same snapshot's task conflict and the retry + # finds the first one's (bed bf32 2026-11-21). + REPL_SNAP_TASK_PREFIX = "repl_snap_task" + + def _repl_snap_task_key(self, cluster_id, snapshot_id): + return f"{self.REPL_SNAP_TASK_PREFIX}/{cluster_id}/{snapshot_id}".encode() + + def _ensure_snapshot_replication_task_tx(self, tr, task, index_list): + claim_key = self._repl_snap_task_key( + task.cluster_id, task.function_params["snapshot_id"]) + raw = tr.get(claim_key).wait() + if raw is not None and raw.present(): + current = BaseModel._read_record(tr, bytes(raw), JobSchedule) + if (current is not None and current.status != JobSchedule.STATUS_DONE + and not current.canceled): + return current.uuid, False + task.updated_at = str(datetime.datetime.now(datetime.UTC)) + self._put_tx(tr, task, index_list) + tr[claim_key] = task.get_db_id().encode() + return task.uuid, True + + def ensure_snapshot_replication_task(self, task: JobSchedule) -> tuple[str, bool]: + """Create ``task`` (an FN_SNAPSHOT_REPLICATION task) unless its + snapshot already has one that is neither done nor canceled -- checked + and written in ONE transaction with the snapshot's claim key, so + concurrent callers never create two tasks for one snapshot. + + bed bf32 (2026-11-21): two replication workers enqueued the same chain + ancestor 100 ms apart; each saw no task in its scan and each wrote + one. The two tasks then ran side by side: one created the landing + volume while the other deleted it as "half-created", leaving a stale + registration on the secondaries that later crashed SPDK. + + Returns ``(uuid of the snapshot's active task, whether it is task)``. + """ + return self._transact(DBController._ensure_snapshot_replication_task_tx, + task, JobSchedule.active_indexes(self.kv_store)) + def _finish_sync_resync_tx(self, tr, task_key, lvs_name, seq_limit, result, duration_sec, task_indexes): task = BaseModel._read_record(tr, task_key, JobSchedule) diff --git a/simplyblock_core/migration_space.py b/simplyblock_core/migration_space.py new file mode 100644 index 0000000000..2d91c5a93a --- /dev/null +++ b/simplyblock_core/migration_space.py @@ -0,0 +1,195 @@ +"""A migration must never fill its SOURCE cluster by itself. + +Bed bf29 (2026-11-20): five volumes of one source cluster ran their cutover +endgame at the same time, under sustained client overwrite. Every shrink +round takes a snapshot of the source volume, and every snapshot pins what the +client overwrites after it; the superseded ones were deleted (async on the +primary, sync on the secondaries) more slowly than new ones were taken. The +source went 41% -> 82% -> 99% in 16 minutes (~440 MB/s of pinned data) and +ran out of space. + +Three rules bound what a migration consumes on its source. Only the +migration's OWN snapshots are held back by them -- never a client write, and +never a client volume creation: the capacity thresholds (cap_warn/cap_crit) +stay alarms, and a cluster is full at 100 %. + +1. One endgame per source cluster at a time + (``REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER``; tasks_runner_replication_final. + _acquire_lvs_claim). The others keep replicating on their ordinary cadence + meanwhile -- an endgame queued for the slot holds no lvstore. +2. An endgame takes its next round snapshot only once the deletions of the + volume's earlier snapshots have completed (:func:`deletions_pending`). +3. No migration snapshot -- endgame round or ordinary cadence of a + migration-mode volume -- while the source's free space is under the + headroom estimate (:func:`space_hold`). +""" +import logging +import time + +from simplyblock_core import constants +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.storage_node import StorageNode + +logger = logging.getLogger(__name__) + + +def source_cluster_id(db, lvol, default=""): + """The cluster *lvol* lives on (its host node's), else *default*.""" + try: + return db.get_storage_node_by_id(lvol.node_id).cluster_id or default + except Exception: # noqa: BLE001 + return default + + +def cluster_free(db, cluster_id, now=None): + """``(free bytes, total bytes, "")`` from the cluster's latest capacity + record, or ``(None, None, why)`` when there is no usable one: none at + all, unreadable, no total, or older than REPL_MIGRATION_CAPACITY_STALE_SEC + (the collector writes one every DEV_STAT_COLLECTOR_INTERVAL_SEC; a record + much older than that says the collector is down, not that space is + free).""" + now = time.time() if now is None else now + try: + cluster = db.get_cluster_by_id(cluster_id) + records = db.get_cluster_capacity(cluster, 1) + except Exception as e: # noqa: BLE001 + return None, None, f"capacity of cluster {cluster_id} unreadable: {e}" + if not records: + return None, None, f"no capacity record for cluster {cluster_id}" + rec = records[0] + total = int(getattr(rec, "size_total", 0) or 0) + if total <= 0: + return None, None, f"capacity record of cluster {cluster_id} has no total" + age = now - float(getattr(rec, "date", 0) or 0) + if age > constants.REPL_MIGRATION_CAPACITY_STALE_SEC: + return None, None, (f"capacity record of cluster {cluster_id} is {int(age)}s " + f"old (limit {constants.REPL_MIGRATION_CAPACITY_STALE_SEC}s)") + used = int(getattr(rec, "size_used", 0) or 0) + return max(0, total - used), total, "" + + +def dirty_bytes(db, lvol): + """Bytes written to *lvol* since its newest snapshot, or None. + + The dirty bitmap of the volume's top bdev + (``bdev_lvol_dirty_bitmap_info``). Every online member is asked and the + largest complete count wins: only the leader's is the real one, and an + over-estimate only makes the hold more careful.""" + from simplyblock_core.controllers.replication_delta import _members + best = None + try: + members = _members(db, lvol) + except Exception: # noqa: BLE001 + return None + for node in members: + if getattr(node, "status", None) != StorageNode.STATUS_ONLINE: + continue + try: + info = node.rpc_client(timeout=constants.REPL_FINAL_DELTA_COUNTS_TIMEOUT_SEC, + retry=0).bdev_lvol_dirty_bitmap_info(lvol.top_bdev) + except Exception: # noqa: BLE001 + continue + if isinstance(info, dict) and info.get("tracking") and info.get("complete"): + n = int(info.get("dirty_bytes") or 0) + best = n if best is None else max(best, n) + return best + + +def headroom_floor(total): + """The free space a migration snapshot always leaves: REPL_MIGRATION_MIN_FREE_PCT + of the cluster.""" + return int(total * constants.REPL_MIGRATION_MIN_FREE_PCT / 100) + + +def required_free(total, dirty): + """Free space a migration snapshot needs: REPL_MIGRATION_HEADROOM_FACTOR + times what the snapshot pins at once (the bytes written since the + volume's last snapshot), and never less than :func:`headroom_floor`. + + Why x2: the new snapshot pins those bytes the moment it is taken, and + while its round transfers the client overwrites about as much again -- + new allocations that the previous snapshot, still pending deletion, keeps + from being reused. The floor covers what one volume's count cannot see: + the other volumes' writes in between, and that the capacity record is up + to one collector interval old (at bf29's 440 MB/s, 15 s are 6.6 GB).""" + return max(headroom_floor(total), + int(constants.REPL_MIGRATION_HEADROOM_FACTOR * max(0, dirty or 0))) + + +def assess(db, lvol, cluster_id, now=None): + """``("ok" | "low" | "unknown", detail)`` for a migration snapshot of + *lvol* on *cluster_id*. Asks SPDK for the volume's dirty count only when + the answer depends on it.""" + free, total, why = cluster_free(db, cluster_id, now) + if free is None: + return "unknown", why + floor = headroom_floor(total) + if free < floor: + return "low", (f"{free >> 30} GiB free on the source cluster, under the " + f"{constants.REPL_MIGRATION_MIN_FREE_PCT}% floor ({floor >> 30} GiB)") + # The dirty count cannot exceed the volume: with that much room there is + # nothing to ask. + size = int(getattr(lvol, "size", 0) or 0) + if size and free >= max(floor, constants.REPL_MIGRATION_HEADROOM_FACTOR * size): + return "ok", "" + dirty = dirty_bytes(db, lvol) + if dirty is None: + # Unknown dirty count: the worst case is the whole volume. + dirty = size + need = required_free(total, dirty) + if free < need: + return "low", (f"{free >> 30} GiB free on the source cluster, under the " + f"{need >> 30} GiB headroom ({constants.REPL_MIGRATION_HEADROOM_FACTOR}x " + f"{dirty >> 20} MiB written since the last snapshot, floor " + f"{floor >> 30} GiB)") + return "ok", "" + + +def space_hold(db, lvol, cluster_id, unknown_since=0, now=None): + """Whether a migration snapshot of *lvol* must wait for space. + + Returns ``(reason, unknown_since)``: an empty reason means take it. A + capacity that cannot be read holds the snapshot for at most + REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC from *unknown_since* (the caller + keeps it and hands it back) and then lets it through, loudly: a broken + stats collector must not stop every migration for good. Space that is + known to be short holds for as long as it lasts -- the endgame's own + shrink deadline bounds that wait, and a cadence snapshot just waits.""" + now = time.time() if now is None else now + verdict, detail = assess(db, lvol, cluster_id, now) + if verdict == "ok": + return "", 0 + if verdict == "low": + return f"migration snapshot of {lvol.get_id()} deferred: {detail}", 0 + since = unknown_since or now + waited = now - since + if waited >= constants.REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC: + logger.warning("migration snapshot of %s: source capacity unknown for %ds " + "(%s); proceeding without the free-space check", + lvol.get_id(), int(waited), detail) + return "", since + return (f"migration snapshot of {lvol.get_id()} deferred: {detail}; " + f"proceeding without the check in " + f"{int(constants.REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC - waited)}s"), since + + +def deletions_pending(db, lvol): + """Ids of *lvol*'s snapshots whose deletion has not completed yet + (status in_deletion). A soft-deleted snapshot (``deleted`` but online, + held by a clone) is not pending: nothing is deleting it.""" + try: + snaps = db.get_snapshots_by_lvol_id(lvol.get_id()) + except Exception as e: # noqa: BLE001 + logger.warning("Cannot read the snapshots of %s: %s", lvol.get_id(), e) + return [] + return [s.get_id() for s in snaps if s.status == SnapShot.STATUS_IN_DELETION] + + +def space_low_event(lvol, reason): + """A warning-level cluster event: a migration snapshot waits for space.""" + try: + from simplyblock_core.controllers import lvol_events + lvol_events.migration_space_low(lvol, reason) + except Exception as e: # noqa: BLE001 + logger.warning("Could not log the migration space event for %s: %s", + lvol.get_id(), e) diff --git a/simplyblock_core/services/snapshot_monitor.py b/simplyblock_core/services/snapshot_monitor.py index 0251602c65..96beba07d5 100644 --- a/simplyblock_core/services/snapshot_monitor.py +++ b/simplyblock_core/services/snapshot_monitor.py @@ -6,6 +6,7 @@ from simplyblock_core import ( constants, db_controller, + migration_space, migration_turn, replication_spare, utils, @@ -910,6 +911,39 @@ def _cutover_endgame_lvol_ids(cluster_id): return ids +# Per volume: since when its source capacity could not be read, and the +# volumes whose cadence currently waits for space (one event per wait). +_space_unknown_since: dict = {} +_space_low: set = set() + + +def _migration_space_hold(lvol, cluster_id, now_ts): + """Why the cadence snapshot of a migration-mode volume must wait for free + space on its source cluster, or "" (rule 3 of + simplyblock_core/migration_space.py, bed bf29 2026-11-20). Only a + migration's own snapshots wait: a volume replicating for DR is not + affected, and neither is any client write.""" + if getattr(lvol, "replication_mode", "") != "migration": + return "" + vid = lvol.get_id() + why, since = migration_space.space_hold( + db, lvol, cluster_id, _space_unknown_since.get(vid, 0), now_ts) + if since: + _space_unknown_since[vid] = since + else: + _space_unknown_since.pop(vid, None) + if why: + if vid not in _space_low: + _space_low.add(vid) + migration_space.space_low_event(lvol, why) + return why + if vid in _space_low: + _space_low.discard(vid) + logger.info("Cadence snapshots of migrating lvol %s resume: source space " + "recovered", vid) + return "" + + def take_due_internal_snapshots(cluster_id, now_ts): """Create an internal snapshot for every replicated volume whose interval has elapsed. The snapshot's creation auto-enqueues a replication task. @@ -1022,6 +1056,14 @@ def _group_id_of(lvol): break if held: continue + for lv in members: + held = _migration_space_hold(lv, cluster_id, now_ts) + if held: + logger.warning("Skipping group snapshot for group %s: %s", + group_id, held) + break + if held: + continue group = db.get_consistency_group_by_id(group_id) # A group is pinned to one LVS: one lookup defers it as a whole. if sync and _cadence_waits_for_leader(group.node_id, group.lvs_name, leaderless): @@ -1063,6 +1105,10 @@ def _group_id_of(lvol): logger.warning("Skipping internal replication snapshot for lvol %s: " "replication held: %s", lvol.get_id(), held) continue + held = _migration_space_hold(lvol, cluster_id, now_ts) + if held: + logger.warning("Skipping internal replication snapshot: %s", held) + continue name = f"repl_internal_{lvol.get_id()[:8]}_{now_ts}" logger.info(f"Taking internal replication snapshot for lvol {lvol.get_id()}: {name}") _id, err = replication_delta.add_snapshot( diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 1e5e2c1a01..cee1f50633 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -1,8 +1,10 @@ +import errno import logging import threading import time import uuid from concurrent.futures import ThreadPoolExecutor +from datetime import datetime from simplyblock_core import ( constants, @@ -48,6 +50,12 @@ _HUB_LOCKS: dict = {} _HUB_LOCKS_GUARD = threading.Lock() +# Names of the landing volumes this process is creating right now +# (_create_landing_volume). A record wearing one of them is in_creation +# because its create is still running here, not because an attempt died. +_LANDING_CREATING: dict = {} # name -> number of creates in flight +_LANDING_CREATING_GUARD = threading.Lock() + def _hub_lock(source_node_id, hub_node_id): with _HUB_LOCKS_GUARD: @@ -633,7 +641,138 @@ def _await_transfer_completion(task, snapshot, snode): def _create_landing_volume(name, snapshot, remote_node, remote_pool_uuid): - """add_lvol_ha for the landing copy of *snapshot*: (lvol id, error).""" + """add_lvol_ha for the landing copy of *snapshot*: (lvol id, error). + + The name is registered in _LANDING_CREATING while add_lvol_ha runs, so + the half-created-landing cleanup never takes a volume under creation for + a dead attempt's leftover (bed bf32 2026-11-21).""" + with _LANDING_CREATING_GUARD: + _LANDING_CREATING[name] = _LANDING_CREATING.get(name, 0) + 1 + try: + return _create_landing_volume_unguarded( + name, snapshot, remote_node, remote_pool_uuid) + finally: + with _LANDING_CREATING_GUARD: + left = _LANDING_CREATING.get(name, 1) - 1 + if left > 0: + _LANDING_CREATING[name] = left + else: + _LANDING_CREATING.pop(name, None) + + +def _landing_age_sec(lvol, now=None): + """Seconds since *lvol*'s record was created; None when unknown.""" + try: + created = datetime.fromisoformat(str(lvol.create_dt)) + except (TypeError, ValueError): + return None + return ((datetime.now() if now is None else now) - created).total_seconds() + + +def _landing_creation_live(task, existing, rep_name): + """Why the in_creation landing volume *existing* may still be under + creation -- "" only when it is provably abandoned. + + bed bf32 (2026-11-21): a second task of the same snapshot took the + landing volume another worker was creating for a dead attempt's leftover + and deleted it; the delete ran on the leader while the creator was still + registering the volume on the secondary and tertiary, which kept a stale + registration that SPDK later crashed on. A landing volume is abandoned + only when nothing here is creating it, no other live task of the snapshot + may be creating it, and it has been in_creation longer than any create + takes (REPL_LANDING_ABANDON_SEC, which also covers a create in another + process, e.g. an old runner pod during a rolling update).""" + with _LANDING_CREATING_GUARD: + if _LANDING_CREATING.get(rep_name): + return "its create is running in this runner" + try: + siblings = _live_siblings(task) + except Exception as e: # noqa: BLE001 + return f"the task table cannot be read ({e})" + if siblings: + return f"task {siblings[0].uuid} of the same snapshot is live" + age = _landing_age_sec(existing) + if age is not None and age < constants.REPL_LANDING_ABANDON_SEC: + return (f"it is only {int(age)}s old " + f"(abandoned after {constants.REPL_LANDING_ABANDON_SEC}s)") + return "" + + +def _landing_member_nodes(lvol): + """Every node the landing volume may be registered on: its own nodes and + the members of its lvstore (a register can land before the record lists + the node).""" + ids = [] + for nid in [lvol.node_id] + list(lvol.nodes or []): + if nid and nid not in ids: + ids.append(nid) + try: + primary = db.get_storage_node_by_id(lvol.node_id) + for nid in (getattr(primary, "secondary_node_id", ""), + getattr(primary, "tertiary_node_id", "")): + if nid and nid not in ids: + ids.append(nid) + except KeyError: + pass + return ids + + +def _confirm_landing_gone(lvol, member_ids): + """After a half-created landing volume's delete: confirm its bdev is gone + on every member, sync-deleting a registration left behind. Returns the + members where removal is not confirmed; each of them gets a durable + sync-delete task, so no registration is left behind untracked. + + A record still present (in_deletion) is the lvol monitor's to finish; + nothing is probed then.""" + try: + db.get_lvol_by_id(lvol.get_id()) + return [] + except KeyError: + pass + if not lvol.lvol_bdev or not lvol.lvs_name: + return [] + bdev = f"{lvol.lvs_name}/{lvol.lvol_bdev}" + unconfirmed = [] + for nid in member_ids: + try: + node = db.get_storage_node_by_id(nid) + except KeyError: + continue # gone with its lvstore: owes nothing + confirmed = False + if node.status == StorageNode.STATUS_ONLINE: + try: + rpc = node.rpc_client(timeout=5, retry=2) + if rpc.bdev_get(bdev) is None: + confirmed = True + else: + logger.warning("Landing volume %s left %s registered on %s; " + "deleting it there", lvol.get_id(), bdev, nid[:8]) + try: + rpc.delete_lvol(bdev, sync=True) + except RPCException as e: + if getattr(e, "code", None) != -errno.ENODEV: + raise + confirmed = rpc.bdev_get(bdev) is None + except Exception as e: # noqa: BLE001 + logger.warning("Cannot confirm %s is gone on %s: %s", bdev, nid[:8], e) + if not confirmed: + unconfirmed.append(nid) + try: + from simplyblock_core.controllers import tasks_controller + tasks_controller.add_lvol_sync_del_task( + node.cluster_id, nid, bdev, lvol.node_id) + except Exception as e: # noqa: BLE001 + logger.error("Could not queue the delete of %s on %s: %s", + bdev, nid[:8], e) + if unconfirmed: + logger.error("Removal of landing volume %s (%s) not confirmed on %s; " + "a durable sync delete is queued for each", + lvol.get_id(), bdev, [n[:8] for n in unconfirmed]) + return unconfirmed + + +def _create_landing_volume_unguarded(name, snapshot, remote_node, remote_pool_uuid): # Carry the source volume's subsystem-packing capacity onto the landing # copy. Without it the copy defaults to a one-namespace subsystem, which # can never be joined by later namespaced volumes -- so every replicated @@ -656,6 +795,11 @@ def _create_landing_volume(name, snapshot, remote_node, remote_pool_uuid): lvol_priority_class=landing_class) +# = tasks_runner_replication_final.DIRECT_ROUND_PREFIX; a copy, so this module +# does not load the cutover runner (a test keeps the two equal). +_DIRECT_ROUND_PREFIX = "repl_round_" + + def _endgame_next_snap_name(lvol_id, cluster_id): """The name a converging cutover of *lvol_id* gives its NEXT shrink snapshot (next_shrink_snap_name), or "".""" @@ -689,6 +833,16 @@ def _prestage_next_landing(snapshot, remote_lv): next_name = _endgame_next_snap_name(lvol_id, snapshot.cluster_id) if not next_name: return None + # A direct round's snapshot is never replicated: its pages go straight + # into the cutover clone, so nothing ever adopts REP_, and the + # endgame's discard only knows the name of the round after it. Each + # such pre-stage left a full-size internal volume on the target for + # good (bed bf29 2026-11-20: REP_repl_round_572cd23c / 1333e5d1 / + # 41e70a17 on cluster 225e0e10, staged 03:53 / 04:11 / 07:47 by the + # transfer of a cadence snapshot still in flight when the endgame + # entered direct mode, never removed). + if next_name.startswith(_DIRECT_ROUND_PREFIX): + return None rep_name = f"REP_{next_name}" try: db.get_lvol_by_name(rep_name, include_deleted=True) @@ -946,6 +1100,9 @@ def process_snap_replicate_start(task, snapshot): logger.warning(f"Landing volume {existing.get_id()} ({rep_name}) has size " f"{existing.size}, the snapshot {snapshot.size}; replacing it") existing.status = LVol.STATUS_IN_CREATION # -> the delete branch below + replacing = True + else: + replacing = False if existing is not None: if existing.status == LVol.STATUS_ONLINE: logger.info(f"Adopting landing volume {existing.get_id()} " @@ -958,13 +1115,24 @@ def process_snap_replicate_start(task, snapshot): _suspend_for_retry(task, f"stale landing volume {rep_name} still deleting, retrying") return else: + # Only a provably abandoned one (bed bf32 2026-11-21: the + # volume deleted here was being created by another worker). + busy = "" if replacing else _landing_creation_live(task, existing, rep_name) + if busy: + _suspend_for_retry( + task, f"landing volume {rep_name} is in_creation and may " + f"still be under creation ({busy}); waiting", + count=False, level=logging.INFO) + return logger.warning(f"Deleting half-created landing volume " f"{existing.get_id()} ({rep_name}, status " f"{existing.status}) from an interrupted attempt") + members = _landing_member_nodes(existing) try: lvol_controller.delete_lvol(existing, force_delete=True) except Exception as e: logger.error(f"Failed to clear stale landing volume {rep_name}: {e}") + _confirm_landing_gone(existing, members) _suspend_for_retry(task, f"cleared stale landing volume {rep_name}, retrying") return @@ -1001,6 +1169,10 @@ def process_snap_replicate_start(task, snapshot): # recovery run from here, during a forced shutdown of the target primary, # left LVS_10 with two leaders and corrupted metadata). A leaderless # target is waited out: the transfer stays suspended and the RPO grows. + # A wait that outlasts REPL_TARGET_LEADERLESS_RECOVER_SEC with every + # target member online asks the control plane's guarded recovery + # (_ask_target_leadership_recovery); that recovery, not replication, + # decides whether a grant is safe. remote_lv_node, lead_why = _stable_target_leader(remote_lv) if remote_lv_node is None: # Waiting, not failing: the target's leadership is not this transfer's @@ -1468,8 +1640,10 @@ def _wait_for_target_leader(task, remote_lv, lead_why): IO, so no member promotes itself when its primary goes down; the node monitor's proven takeover gives it a leader, during the down member's restart too while that member's SPDK is down (bf24 2026-11-14, LVS_22: - the transfer waited ~110 s for the crashed primary to come back). The - wait is polled every REPL_RETRY_BACKOFF_BASE_SEC (step 0, whatever the + the transfer waited ~110 s for the crashed primary to come back). With + every member online the wait asks the control plane's guarded recovery + (_ask_target_leadership_recovery) once it lasts + REPL_TARGET_LEADERLESS_RECOVER_SEC. The wait is polled every REPL_RETRY_BACKOFF_BASE_SEC (step 0, whatever the task's earlier retries), costs no retry, and raises a REPLICATION_STALLED event once it lasts REPL_TARGET_LEADERLESS_ALERT_SEC, then at most every REPL_MERGE_ALERT_EVERY_SEC.""" @@ -1492,14 +1666,53 @@ def _wait_for_target_leader(task, remote_lv, lead_why): except Exception as e: # noqa: BLE001 logger.error("Replication task %s: could not log the leaderless-target " "event: %s", task.uuid, e) + _ask_target_leadership_recovery(task, remote_lv, waited, now) _suspend_for_retry(task, msg, backoff=True, count=_switches_exhausted(task), backoff_step=0) +def _ask_target_leadership_recovery(task, remote_lv, waited, now): + """Once a leaderless-target wait has lasted REPL_TARGET_LEADERLESS_RECOVER_SEC, + ask the control plane to restore the target's leadership, at most every + REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC per task. + + Replication still grants nothing itself and never forces: the recovery + (storage_node_ops.restore_healthy_lvs_leadership) acts only when EVERY + member of the target lvstore is online, answers and reports non-leader, + and then through the guarded reload-then-grant every object path uses. + A member that is down or restarting leaves it to the node monitor's + offline takeover and to the restart, as before (bf8 2026-11-02: the + unguarded grant from here during a forced shutdown made two leaders). + bf32 2026-11-21: both target lvstores (LVS_7, LVS_10) lost their leader to + an SPDK self-demotion on a failed write; every member was back online + minutes later, nothing restored the leadership, and the transfers waited + 76 min.""" + params = task.function_params + if waited < constants.REPL_TARGET_LEADERLESS_RECOVER_SEC: + return + tried = params.get("target_recover_tried_at") or 0 + if now - tried < constants.REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC: + return + params["target_recover_tried_at"] = now + try: + from simplyblock_core import storage_node_ops + leader, why = storage_node_ops.restore_healthy_lvs_leadership( + remote_lv.node_id, remote_lv.lvs_name, + reason=f"replication task {task.uuid} waited {waited}s") + except Exception as e: # noqa: BLE001 + logger.error("Replication task %s: leadership recovery of %s failed: %s", + task.uuid, remote_lv.lvs_name, e) + return + logger.info("Replication task %s: leadership recovery of target %s: %s", + task.uuid, remote_lv.lvs_name, why if leader is None + else f"{why} (leader {leader.get_id()})") + + def _target_leader_found(task): """End a leaderless-target wait: log how long it lasted.""" since = task.function_params.pop("target_leaderless_since", None) task.function_params.pop("target_leaderless_alerted_at", None) + task.function_params.pop("target_recover_tried_at", None) if since: logger.info("Replication task %s: target leadership settled after %ds without " "a leader", task.uuid, int(time.time() - since)) @@ -3273,15 +3486,62 @@ def _wait(seconds, wake): _check_endgame_wake(wake) +def _live_siblings(task): + """The other live (not done, not canceled) replication tasks of *task*'s + snapshot, through the repl_snapshot_id index (a point read, not a scan + of the never-pruned task table).""" + snapshot_id = (task.function_params or {}).get("snapshot_id") + if not snapshot_id: + return [] + return [t for t in db.get_replication_tasks_for_snapshot(snapshot_id) + if t.uuid != task.uuid + and (not task.cluster_id or t.cluster_id == task.cluster_id) + and t.function_name == JobSchedule.FN_SNAPSHOT_REPLICATION + and t.status != JobSchedule.STATUS_DONE and t.canceled is False] + + +def _task_precedence(t): + """Sort key: which of several live tasks of one snapshot may run. A + transfer already running first, then one that owns a landing volume, + then the oldest; the uuid breaks ties, so every worker picks the same.""" + params = t.function_params or {} + return (t.status != JobSchedule.STATUS_RUNNING, + not params.get("remote_lvol_id"), + int(t.date or 0), t.uuid) + + +def _yields_to_sibling(task): + """The task of the same snapshot *task* must wait for, or None. + + One snapshot has one replication task (_add_task claims it atomically), + but a duplicate written before that claim existed -- or by any other + path -- must not run beside it: two runs of one snapshot each create and + delete the same REP_* landing volume (bed bf32 2026-11-21). Of the live + tasks of a snapshot exactly one runs, by _task_precedence; the others + wait until it is done, then find the snapshot replicated.""" + siblings = _live_siblings(task) + if not siblings: + return None + first = min(siblings + [task], key=_task_precedence) + return None if first.uuid == task.uuid else first + + def _dispatch(task): """Run one replication task the way a pass does; task_runner's result (False on a failure, which the caller follows with a short pause).""" - if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED]: - for t in db.get_job_tasks(task.cluster_id): - if t.function_name == JobSchedule.FN_SNAPSHOT_REPLICATION and t.function_params["snapshot_id"] == task.function_params['snapshot_id']: - if t.status == JobSchedule.STATUS_RUNNING and t.canceled is False: - logger.info("replication task found for same snapshot, retry") - return True + if task.status != JobSchedule.STATUS_DONE and not task.canceled: + try: + first = _yields_to_sibling(task) + except Exception as e: # noqa: BLE001 + # Unknown is not "alone": run it on a later pass. + logger.warning("Cannot read the tasks of snapshot %s: %s", + task.function_params.get("snapshot_id"), e) + return False + if first is not None: + logger.info("Replication task %s waits: task %s of the same snapshot %s " + "runs first", task.uuid, first.uuid, + task.function_params.get("snapshot_id")) + return True if task.status == JobSchedule.STATUS_DONE: return True # Re-read the task in case cancel changed it. If it has @@ -3350,6 +3610,7 @@ def __init__(self, size): if self.size else None) self._guard = threading.Lock() self._inflight: dict = {} # task uuid -> Future + self._inflight_snaps: dict = {} # snapshot id -> task uuid running it self._not_before: dict = {} # task uuid -> earliest next submit def paused(self, task): @@ -3367,11 +3628,18 @@ def submit(self, task): result then, True otherwise.""" if self._pool is None: return _dispatch(task) + snap_id = (task.function_params or {}).get("snapshot_id") or "" with self._guard: if task.uuid in self._inflight: return True + # One snapshot on one worker: a second task of the same snapshot + # never runs beside the first (bed bf32 2026-11-21). + if snap_id and self._inflight_snaps.get(snap_id, task.uuid) != task.uuid: + return True if time.time() < self._not_before.get(task.uuid, 0.0): return True + if snap_id: + self._inflight_snaps[snap_id] = task.uuid self._inflight[task.uuid] = self._pool.submit(self._run, task) return True @@ -3385,6 +3653,9 @@ def _run(self, task): finally: with self._guard: self._inflight.pop(task.uuid, None) + snap_id = (task.function_params or {}).get("snapshot_id") or "" + if snap_id and self._inflight_snaps.get(snap_id) == task.uuid: + del self._inflight_snaps[snap_id] if not res: self._not_before[task.uuid] = ( time.time() + constants.REPL_RUNNER_RETRY_PAUSE_SEC) diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index 0d73802d63..399629a807 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -1843,6 +1843,84 @@ def _take_over_leaderless_lvstores(snode): lvs_name, leader.get_id(), snode.get_id(), snode.status) +#: lvstore key -> monotonic time of the last healthy-leaderless probe. +_healthy_leaderless_checked: dict[tuple, float] = {} +#: lvstore key -> monotonic time it was first seen leaderless with every +#: member online (cleared by any other observation). +_healthy_leaderless_since: dict[tuple, float] = {} +#: lvstore keys whose recovery runs right now (a background thread each). +_healthy_leaderless_running: set = set() +_healthy_leaderless_lock = threading.Lock() +HEALTHY_LEADERLESS_CHECK_INTERVAL_SEC = 30 +HEALTHY_LEADERLESS_CONFIRM_SEC = 60 + + +def _restore_leadership_if_leaderless(snode): + """Give the own lvstore of ONLINE primary ``snode`` its leader back when it + has none while every member is online and healthy. + + Leadership is placed by the restart, creation and activation flows, and + the offline takeover (_take_over_leaderless_lvstores) covers a DOWN + member; nothing covered a leaderless lvstore whose members are all up. + bf32 2026-11-21 (R6): with two of five nodes' SPDK gone, a write to LVS_7 + and LVS_10 failed and each primary's SPDK dropped the leadership itself + (it takes it back only on the next IO, and both were replication targets + without client IO). The restarts of the other members then rebuilt them + as non-leaders behind the demoted primaries, and the lvstores stayed + leaderless for 76 min with every member online. + + Probed at most every HEALTHY_LEADERLESS_CHECK_INTERVAL_SEC by the + primary's own monitor thread (one lvstore, three get_lvstore RPCs); acted + on only after it was seen leaderless for HEALTHY_LEADERLESS_CONFIRM_SEC, + so a hand-off in flight is not mistaken for a lost leader. The grant is + storage_node_ops.restore_healthy_lvs_leadership (the guarded + reload-then-grant of _recover_leaderless_lvs, which also refuses while a + restart or port-allow task is active on a member), run on a background + thread: it waits out a safety window and must not stall this node's + liveness checks. Not on sync-replication or two-node arbitrated clusters + (healthy_lvs_leadership_state).""" + if not snode.lvstore or snode.is_secondary_node: + return + key = (snode.cluster_id, snode.lvstore) + now = time.monotonic() + if now - _healthy_leaderless_checked.get(key, -1e9) < HEALTHY_LEADERLESS_CHECK_INTERVAL_SEC: + return + _healthy_leaderless_checked[key] = now + with _healthy_leaderless_lock: + if key in _healthy_leaderless_running: + return + state, _, why = storage_node_ops.healthy_lvs_leadership_state(snode.get_id()) + if state != "leaderless": + _healthy_leaderless_since.pop(key, None) + return + since = _healthy_leaderless_since.setdefault(key, now) + if now - since < HEALTHY_LEADERLESS_CONFIRM_SEC: + logger.warning("%s; restoring its leadership if it stays so for %ss", + why, HEALTHY_LEADERLESS_CONFIRM_SEC) + return + with _healthy_leaderless_lock: + if key in _healthy_leaderless_running: + return + _healthy_leaderless_running.add(key) + + def _run(): + try: + leader, result = storage_node_ops.restore_healthy_lvs_leadership( + snode.get_id(), reason=f"leaderless for {int(time.monotonic() - since)}s, " + f"seen by the node monitor") + logger.info("Leadership recovery of %s: %s", snode.lvstore, result) + if leader is not None: + _healthy_leaderless_since.pop(key, None) + except Exception as e: # noqa: BLE001 + logger.error("Leadership recovery of %s failed: %s", snode.lvstore, e) + finally: + with _healthy_leaderless_lock: + _healthy_leaderless_running.discard(key) + + threading.Thread(target=_run, daemon=True, + name=f"lvs-leadership-{snode.lvstore}").start() + + def _ana_failover_rearm(node_id) -> None: """Forget the one-shot marker so the next offline episode re-triggers.""" _ana_failover_applied.discard(node_id) @@ -3041,6 +3119,14 @@ def check_node(snode): ) storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_ONLINE) readmit_devices_after_node_online(snode.get_id()) + return + + if snode.status == StorageNode.STATUS_ONLINE: + try: + _restore_leadership_if_leaderless(snode) + except Exception as e: # noqa: BLE001 + logger.error("Leadership check of %s on %s failed: %s", + snode.lvstore, snode.get_id(), e) def readmit_devices_after_node_online(node_id): diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index f03dbe23ea..6f9af8c6fd 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -25,7 +25,13 @@ from datetime import datetime from typing import Any, NoReturn -from simplyblock_core import constants, db_controller, utils, xfer_timing +from simplyblock_core import ( + constants, + db_controller, + migration_space, + utils, + xfer_timing, +) from simplyblock_core.controllers import ns_host_guard from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol, LVolReplication @@ -137,7 +143,8 @@ def _release_lvs_claim(task): before this runs. """ _requeue_direct_rounds(task) - drop_params(task, "cutover_lvs", "cutover_group", "throttle_tried") + drop_params(task, "cutover_lvs", "cutover_group", "cutover_cluster", + "cutover_claimed_at", "throttle_tried") def _throttle_cap(task): @@ -360,7 +367,7 @@ def _delete_source_if_requested(task): while True: blocked = ns_host_guard.removal_blocked( getattr(src_lvol, "nqn", ""), ns_host_guard.volume_node_ids(src_lvol), - _target_node_ids(task.function_params), + _live_target_node_ids(src_lvol, task.function_params), f"deleting migrated-away source {src_lvol_id}") if not blocked or time.time() >= deadline: break @@ -578,6 +585,10 @@ def _acquire_lvs_claim(task, lvol, tasks): int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC)) raise TaskDefer(f"queued for lvstore {lvs_name} behind " f"{owner_lvol[:8]} (delta already converged)") + cluster_id = migration_space.source_cluster_id(db, lvol, task.cluster_id) + busy = _cluster_endgame_slots_taken(task, tasks, own_group, cluster_id) + if busy: + raise TaskDefer(busy) xfer_timing.stamp("lvs_claim_acquired", lvol=lvol.get_id(), lvs=lvs_name, round=params.get("shrink_round")) # Committed immediately: the claim has to be visible to the other hosts' @@ -585,7 +596,93 @@ def _acquire_lvs_claim(task, lvol, tasks): # endgame_started_at survives a released-and-retaken claim: the plan # forecast measures a cutover from its FIRST claim to done. extra = {} if params.get("endgame_started_at") else {"endgame_started_at": time.time()} - return _record(task, cutover_lvs=lvs_name, cutover_group=own_group, **extra) + task = _record(task, cutover_lvs=lvs_name, cutover_group=own_group, + cutover_cluster=cluster_id, cutover_claimed_at=time.time(), **extra) + return _confirm_cluster_endgame_slot(task, own_group, cluster_id) + + +def _endgame_unit(t): + """What one endgame slot holds: a consistency group cuts over as one.""" + params = t.function_params or {} + return params.get("cutover_group") or t.get_id() + + +def _cluster_endgame_holders(tasks, cluster_id): + """The FN_REPLICATION_FINAL tasks of *tasks* that hold an endgame of + *cluster_id* (a claimed lvstore), live ones only.""" + out = [] + for t in tasks or (): + if t.function_name != JobSchedule.FN_REPLICATION_FINAL: + continue + if t.status == JobSchedule.STATUS_DONE or t.canceled: + continue + params = t.function_params or {} + if not params.get("cutover_lvs"): + continue + # a claim from before the cluster was recorded counts where it runs + if (params.get("cutover_cluster") or t.cluster_id) != cluster_id: + continue + out.append(t) + return out + + +def _cluster_endgame_slots_taken(task, tasks, own_group, cluster_id): + """Why this volume may not start an endgame on its source cluster now, + or "" (rule 1 of simplyblock_core/migration_space.py). + + At most REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER endgames run at once per + source cluster, whatever lvstores they are on: each one takes a snapshot + of its volume every round, and bf29 (2026-11-20) ran five at once and + filled the source. A volume that waits here has taken no snapshot and + holds no lvstore, so it keeps replicating on its ordinary cadence. A + consistency-group sibling joins its group's slot.""" + limit = max(1, int(constants.REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER)) + units = {_endgame_unit(t) for t in _cluster_endgame_holders(tasks, cluster_id) + if t.get_id() != task.get_id()} + if own_group and own_group in units: + return "" + if len(units) < limit: + return "" + _record(task, shrink_deadline=( + int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC)) + return (f"queued for an endgame slot of source cluster {cluster_id}: " + f"{len(units)} of {limit} in use") + + +def _confirm_cluster_endgame_slot(task, own_group, cluster_id): + """Re-check the cluster's endgame slots after the claim is committed. + + The pre-check reads the pass's task list, which predates every claim made + in this pass: volumes that all caught up in the same pass would each see + the slots free. Every claimant writes its claim BEFORE this read, so of + two racing claimants at least one sees the other, and both rank the + claims the same way -- by claim time, so a running endgame is never + displaced by a later claim. A claimant ranked past the limit withdraws. + An unreadable task table leaves the pre-check's answer standing.""" + limit = max(1, int(constants.REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER)) + try: + fresh = db.get_job_tasks(task.cluster_id) + except Exception as e: # noqa: BLE001 + logger.warning("cutover for lvol %s: cannot re-check the endgame slots of " + "cluster %s: %s", task.function_params.get("lvol_id"), cluster_id, e) + return task + first: dict[str, tuple[float, str]] = {} + for t in _cluster_endgame_holders(fresh, cluster_id) + [task]: + params = t.function_params or {} + key = (float(params.get("cutover_claimed_at") or 0), t.get_id()) + unit = _endgame_unit(t) + if unit not in first or key < first[unit]: + first[unit] = key + ranked = sorted(first, key=lambda u: first[u]) + own_unit = own_group or task.get_id() + if own_unit in ranked[:limit]: + return task + logger.info("cutover for lvol %s: endgame slot of cluster %s lost to a concurrent " + "claim; withdrawing", task.function_params.get("lvol_id"), cluster_id) + drop_params(task, "cutover_lvs", "cutover_group", "cutover_cluster", + "cutover_claimed_at") + raise TaskDefer(f"queued for an endgame slot of source cluster {cluster_id}: " + f"lost to a concurrent claim") def _await_cutover_proceed(task, params): @@ -657,9 +754,20 @@ def task_runner(task: JobSchedule, tasks): lvol_id, tgt_node.get_id(), tgt_node.status) _record(task, last_error=( f"target node {tgt_node.get_id()[:8]} is {tgt_node.status}")) + # An endgame cannot progress meanwhile, and while it holds the volume + # the volume replicates nothing (bed bf29 R3b): hand it back to the + # ordinary cadence after a short grace (_suspend_endgame_for_target). + if params.get("cutover_lvs") or params.get("shrink_snap_id"): + task = _suspend_endgame_for_target(task, lvol, tgt_node) # Deferred rather than failed: a node being offline is transient, and # burning a retry per pass would exhaust them all before it recovers. raise TaskDefer("target node not online, waiting") + if params.get("target_down_since"): + logger.info("cutover for lvol %s: target node %s is back after %ds", + lvol_id, tgt_node.get_id(), + int(time.time() - params["target_down_since"])) + task = drop_params(task, "target_down_since") or task + params = task.function_params # Cooldown set after a hub-attach failure to give the target node time to # recover and the control plane time to reflect a down state in DB. @@ -756,6 +864,13 @@ def task_runner(task: JobSchedule, tasks): task = drop_params(task, "catchup_since", "catchup_win", "catchup_prev", "catchup_strikes") or task params = task.function_params + # Nothing is claimed while the first round could not be taken anyway + # (pending deletions, a source short of space): holding the lvstore + # and the cluster's endgame slot through that wait would stop the + # other volumes' transfers for nothing. The round checks again. + held, task = _round_snapshot_hold(task, lvol) + if held: + raise TaskDefer(held) task = _acquire_lvs_claim(task, lvol, tasks) # The endgame gets its full convergence budget from here. The deadline # an abandon set (back-off + timeout) also had to cover the prune gate @@ -924,6 +1039,51 @@ def task_runner(task: JobSchedule, tasks): _record_cutover_done(task) +def _suspend_endgame_for_target(task, lvol, tgt_node): + """The target node is down before the cutover clone exists: after + REPL_CUTOVER_TARGET_DOWN_GRACE_SEC, give the endgame up -- without + burning an attempt and without a back-off -- and return the task. + + Bed bf29 R3b (2026-11-20): the target primary was shut down while a + volume was in its endgame. The endgame waited for it (the clone is built + through the target primary), and while it held the volume nothing + replicated it: the endgame takes no snapshot while it waits, and the + cadence snapshot and retention skip a volume in its endgame. No cycle + completed for ~20 min (RPO 1280 s). The wait itself dates from feb45b996 + (2026-08-27), which made it a deferral instead of a burned retry, but + left the endgame -- lvstore claim, round in flight -- in place for it. + + Giving it up drops the endgame's round state and releases the lvstore + claim and the cluster's endgame slot, so the volume falls back to its + ordinary cadence (snapshots + delta transfers to whichever target member + leads, still under the migration space rules) and another volume can + take the slot. The source write cap is lifted: the outage is not the + volume's failure to converge, and may last long. The task stays: once the + target is back, the catch-up gate lets it enter a fresh endgame. + """ + params = task.function_params + now = time.time() + since = params.get("target_down_since") + if not since: + return _record(task, target_down_since=now) + if now - since < constants.REPL_CUTOVER_TARGET_DOWN_GRACE_SEC: + return task + reason = (f"target node {tgt_node.get_id()[:8]} {tgt_node.status} for " + f"{int(now - since)}s before the cutover clone exists") + logger.warning("cutover for lvol %s: endgame suspended: %s; the volume replicates " + "on its ordinary cadence until the target is back", + lvol.get_id(), reason) + xfer_timing.stamp("endgame_suspended_target_down", lvol=lvol.get_id(), + round=params.get("shrink_round")) + task = _record(task, last_error=f"endgame suspended: {reason}") + task = _discard_spare_landing(task) or task + task = drop_params(task, *_ENDGAME_ROUND_PARAMS) or task + _release_lvs_claim(task) + _restore_source_throttle(task) + # (the caller defers right away; the claim's release committed on its own) + return task + + def _prepare_early(task, lvol, src_node, tgt_node): """Build the cutover clone as the endgame starts (REPL_CUTOVER_EARLY_PREPARE). @@ -1236,6 +1396,9 @@ def _take_shrink_snapshot(task, lvol, direct=False): round=params.get("shrink_round", 0)) name = params.get("next_shrink_snap_name") or _new_shrink_snap_name(direct) if direct != name.startswith(DIRECT_ROUND_PREFIX): + # The landing volume pre-staged under the old name would never be + # adopted (bf29: three 10 GiB REP_repl_round_* left on cluster A). + _discard_landing_named(name, lvol.get_id(), task.cluster_id) name = _new_shrink_snap_name(direct) # The pre-recorded name may already exist: an earlier pass created the # snapshot but did not get to record it (bf17, 2026-11-08: "Snapshot name @@ -1249,6 +1412,7 @@ def _take_shrink_snapshot(task, lvol, direct=False): new_snap, err = existing[0].get_id(), None else: if existing: + _discard_landing_named(name, lvol.get_id(), task.cluster_id) name = _new_shrink_snap_name(direct) with xfer_timing.phase("take_shrink_snapshot", lvol=lvol.get_id(), round=params.get("shrink_round", 0) + 1): @@ -1256,7 +1420,9 @@ def _take_shrink_snapshot(task, lvol, direct=False): lvol, name, snap_type=SnapShot.TYPE_INTERNAL, **({"replicate": False} if direct else {})) if err and "must be unique" in str(err): - # taken by something not ours: next pass uses a fresh name + # taken by something not ours: next pass uses a fresh name, and + # what was pre-staged for this one is never adopted + _discard_landing_named(name, lvol.get_id(), task.cluster_id) task = _record(task, next_shrink_snap_name=_new_shrink_snap_name(direct)) if err: # Three values like the success path: every caller unpacks three, and @@ -1337,11 +1503,88 @@ def _direct_last_round(params): return None +def _round_snapshot_hold(task, lvol): + """``(why, task)``: why the next round snapshot must wait, or "". + + Rules 2 and 3 of simplyblock_core/migration_space.py (bed bf29, + 2026-11-20: five endgames' round snapshots pinned ~440 MB/s of + overwritten data and filled the source cluster): + + * Rule 2 -- no new round snapshot while one of the volume's snapshots is + still being deleted: rounds then go only as fast as their predecessors + are released, instead of stacking pending deletions. A deletion still + pending after REPL_ENDGAME_DELETION_WAIT_MAX_SEC is set aside (logged) + and no longer waited for; any newer one is. + * Rule 3 -- no new round snapshot while the source cluster's free space + is under the headroom the snapshot needs (migration_space.space_hold). + + Only this migration's snapshots wait; client IO is never held. The wait + is bounded by the endgame's shrink deadline like any other: an endgame + that cannot snapshot is abandoned, releasing its slot and lvstore. + """ + params = task.function_params + now = time.time() + lvol_id = lvol.get_id() + nxt = params.get("shrink_round", 0) + 1 + + # ---- rule 2: the volume's earlier deletions have completed ----------- # + ignored = set(params.get("deletion_ignored") or ()) + pending = [s for s in migration_space.deletions_pending(db, lvol) if s not in ignored] + if pending: + since = params.get("deletion_wait_since") or now + if not params.get("deletion_wait_since"): + task = _record(task, deletion_wait_since=since) + waited = now - since + if waited < constants.REPL_ENDGAME_DELETION_WAIT_MAX_SEC: + xfer_timing.stamp("round_waits_deletion", lvol=lvol_id, round=nxt, + n=len(pending)) + return (f"round {nxt} snapshot waits for {len(pending)} deletion(s) of " + f"the volume's snapshots to complete " + f"({', '.join(p[:8] for p in pending)})"), task + logger.warning("cutover for lvol %s: snapshot deletion(s) %s still pending after " + "%ds; no longer waiting for them before round %d", + lvol_id, ", ".join(pending), int(waited), nxt) + task = _record(task, deletion_ignored=sorted(ignored | set(pending))) + task = drop_params(task, "deletion_wait_since") or task + elif params.get("deletion_wait_since"): + logger.info("cutover for lvol %s: snapshot deletions completed after %ds; " + "taking round %d", lvol_id, int(now - params["deletion_wait_since"]), nxt) + task = drop_params(task, "deletion_wait_since") or task + params = task.function_params + + # ---- rule 3: free space on the source cluster ------------------------ # + cluster_id = migration_space.source_cluster_id(db, lvol, task.cluster_id) + unknown_since = params.get("space_unknown_since", 0) or 0 + why, new_unknown = migration_space.space_hold(db, lvol, cluster_id, unknown_since, now) + if new_unknown != unknown_since: + task = (_record(task, space_unknown_since=new_unknown) if new_unknown + else drop_params(task, "space_unknown_since") or task) + if why: + if not params.get("space_low_since"): + logger.warning("cutover for lvol %s: %s", lvol_id, why) + migration_space.space_low_event(lvol, why) + task = _record(task, space_low_since=now) + xfer_timing.stamp("round_waits_space", lvol=lvol_id, round=nxt) + return why, task + if params.get("space_low_since"): + logger.info("cutover for lvol %s: source space recovered after %ds; taking " + "round %d", lvol_id, int(now - params["space_low_since"]), nxt) + task = drop_params(task, "space_low_since") or task + return "", task + + def _take_round(task, lvol): """Take the next round's snapshot and, with direct rounds, send it into the cutover clone; (error, task). Direct or not is decided once per endgame, at its first round: a clone holding direct rounds is never - rebased onto ordinary rounds' copies.""" + rebased onto ordinary rounds' copies. + + The snapshot waits (an error, so the caller defers) while the volume's + earlier deletions are pending or the source is short of space + (_round_snapshot_hold).""" + held, task = _round_snapshot_hold(task, lvol) + if held: + return held, task params = task.function_params if "direct_rounds" not in params: direct = _direct_mode_possible(lvol, params) @@ -1699,25 +1942,59 @@ def _discard_spare_landing(task): if not name: return task task = drop_params(task, "next_shrink_snap_name") or task + _discard_landing_named(name, (task.function_params or {}).get("lvol_id"), + task.cluster_id) + return task + + +def _landing_in_transfer(landing_id, cluster_id): + """True when a live replication task transfers into *landing_id*, or when + that cannot be ruled out (the task table is unreadable).""" + try: + tasks = db.get_job_tasks(cluster_id) + except Exception as e: # noqa: BLE001 + logger.warning("Cannot read tasks to check landing volume %s: %s", landing_id, e) + return True + for t in tasks: + if (t.function_name == JobSchedule.FN_SNAPSHOT_REPLICATION + and t.status != JobSchedule.STATUS_DONE and not t.canceled + and (t.function_params or {}).get("remote_lvol_id") == landing_id): + return True + return False + + +def _discard_landing_named(name, lvol_id, cluster_id): + """Delete the landing volume pre-staged for round snapshot *name* + (REP_, snapshot_replication._prestage_next_landing) when that name + will not be used: it exists, is online and no transfer lands in it. + + Nothing else reclaims such a volume: bed bf29 (2026-11-20) left three + 10 GiB REP_repl_round_* landing volumes on cluster A for good. Best + effort, logged loudly when it fails.""" + if not name: + return rep_name = f"REP_{name}" try: spare = db.get_lvol_by_name(rep_name) except KeyError: - return task + return except Exception as e: # noqa: BLE001 logger.error("Could not look up the spare landing volume %s: %s", rep_name, e) - return task - if spare.status == LVol.STATUS_IN_DELETION: - return task + return + if spare.status != LVol.STATUS_ONLINE: + return + if _landing_in_transfer(spare.get_id(), cluster_id): + logger.info("cutover for lvol %s: pre-staged landing volume %s is in use by a " + "transfer; kept", lvol_id, rep_name) + return try: from simplyblock_core.controllers import lvol_controller logger.info("cutover for lvol %s: deleting the unused pre-staged landing " - "volume %s", task.function_params.get("lvol_id"), rep_name) + "volume %s", lvol_id, rep_name) lvol_controller.delete_lvol(spare, force_delete=True) except Exception as e: # noqa: BLE001 logger.error("Could not delete the unused pre-staged landing volume %s " "(%s): %s", rep_name, spare.get_id(), e) - return task def _shrink_round_done(snap_id): @@ -1818,7 +2095,13 @@ def _shrink_step(task, lvol, tasks=()): continue started_at = params.get("shrink_started_at") - if started_at is None: + times = params.get("shrink_round_times") or [] + if params.get("shrink_round_measured") == params["shrink_round"] and times: + # Measured on an earlier pass, which then could not take the next + # snapshot (_round_snapshot_hold, a refused snapshot): the round's + # duration is what was measured then, not the time since. + elapsed = float(times[-1]) + elif started_at is None: # Unmeasurable round (an older task, or one enqueued without the # stamp). Treat it as NOT converged rather than as instant: a # missing measurement must never be read as "the delta is small", @@ -1843,7 +2126,8 @@ def _shrink_step(task, lvol, tasks=()): round=params["shrink_round"], ms=elapsed * 1000.0) task = _record(task, shrink_round_times=round_times, - shrink_round_done_at=time.time()) + shrink_round_done_at=time.time(), + shrink_round_measured=params["shrink_round"]) params = task.function_params # Every round is an endgame round now -- the lvstore is held before the @@ -1973,7 +2257,11 @@ def _final_delta_estimate(params): _ENDGAME_ROUND_PARAMS = ("shrink_snap_id", "shrink_round", "shrink_round_times", "shrink_started_at", "shrink_round_done_at", "direct_rounds", "direct_xfer_node", "direct_submits", - "direct_done_snap", "direct_round_ids") + "direct_done_snap", "direct_round_ids", + # _round_snapshot_hold's waits (bf29) + "shrink_round_measured", + "deletion_wait_since", "deletion_ignored", + "space_low_since", "space_unknown_since") def _abandon_endgame(task, lvol, reason) -> NoReturn: @@ -2121,6 +2409,28 @@ def _target_node_ids(params): return [] +def _live_target_node_ids(src_lvol, params): + """The nodes exporting the volume the source migrated to. + + After the cutover the prepared clone's record is gone: the volume runs + under a new record carrying the same NQN. Bed bf29 (2026-11-20): the + source-delete guard looked the target up by tgt_lvol_id, found nothing, + took every host as stranded and kept the migrated-away source of two + volumes (a full volume leaked per migration, and a same-named volume left + on the cluster the next round migrates back to). So the target is found + by NQN: every other live volume exporting it, plus the clone if it still + exists.""" + ids = list(_target_node_ids(params)) + nqn = getattr(src_lvol, "nqn", "") + if nqn: + for cluster in db.get_clusters(): + for lv in db.get_lvols(cluster.get_id()): + if (lv.get_id() != src_lvol.get_id() and getattr(lv, "nqn", "") == nqn + and lv.status not in (LVol.STATUS_IN_DELETION, LVol.STATUS_DELETED)): + ids.extend(ns_host_guard.volume_node_ids(lv)) + return list(dict.fromkeys(ids)) + + def _source_node_ids(lvol_id): try: return ns_host_guard.volume_node_ids(db.get_lvol_by_id(lvol_id)) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 7e09b48725..907393bc35 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -12638,6 +12638,106 @@ def reconcile_lvs_leadership(cluster_id, nodes, skip_jm_vuids=(), confirmed=(), return restored, leaderless +def healthy_lvs_leadership_state(owner_id, lvs_name=None): + """``(state, members, why)`` for the LVS of ``owner_id`` (its primary). + + ``state`` is ``"leaderless"`` only when the LVS is leaderless but HEALTHY: + the cluster is active or degraded, the owner is an ONLINE primary whose + health check has not failed and whose lvstore is not in a restart window, + every member is ONLINE, every member answers ``get_lvstore``, and none + reports leadership. ``members`` (primary first) is then the set to recover + over. Otherwise ``"led"`` (``members``: the members reporting leadership) + or ``"skip"`` + (``why`` says what does not hold): an offline or restarting member is the + node monitor's offline takeover and the restart's business, a member that + cannot be asked is a POSSIBLE leader. + + Not on sync-replication clusters (the passive triplet must stay + leaderless; the active one is the promote's) nor on two-node arbitrated + ones (the arbiter reconciles there: reconcile_lvs_leadership). + + ``lvs_name``: the LVS the caller means; when ``owner_id`` is not its + primary (a volume's recorded node), the primary is looked up by name.""" + db = DBController() + try: + owner = db.get_storage_node_by_id(owner_id) + except KeyError: + return "skip", [], f"node {owner_id} unknown" + if lvs_name and owner.lvstore != lvs_name: + primary = next((n for n in db.get_storage_nodes_by_cluster_id(owner.cluster_id) + if n.lvstore == lvs_name and not n.is_secondary_node), None) + if primary is None: + return "skip", [], f"no primary of {lvs_name} found" + owner = primary + lvs_name = owner.lvstore + if not lvs_name or owner.is_secondary_node or not owner.jm_vuid: + return "skip", [], f"{owner.get_id()} holds no primary lvstore" + if _sync_site(owner): + return "skip", [], f"{lvs_name}: sync-replication cluster" + if _cluster_two_node_arbitrated(owner.cluster_id, db=db): + return "skip", [], f"{lvs_name}: two-node arbitrated cluster (the arbiter reconciles)" + cluster = db.get_cluster_by_id(owner.cluster_id) + if cluster.status not in (Cluster.STATUS_ACTIVE, Cluster.STATUS_DEGRADED): + return "skip", [], f"{lvs_name}: cluster is {cluster.status}" + if owner.health_check is False or owner.lvstore_status == "in_creation": + return "skip", [], (f"{lvs_name}: primary {owner.get_id()} health_check=" + f"{owner.health_check} lvstore_status={owner.lvstore_status}") + members = [owner] + for nid in (owner.secondary_node_id, owner.tertiary_node_id): + if not nid: + continue + try: + members.append(db.get_storage_node_by_id(nid)) + except KeyError: + return "skip", [], f"{lvs_name}: member {nid} unknown" + not_online = [f"{m.get_id()[:8]} {m.status}" for m in members + if m.status != StorageNode.STATUS_ONLINE] + if not_online: + return "skip", [], f"{lvs_name}: member(s) not online: {', '.join(not_online)}" + leaders, unknown = _lvs_leader_probe(members, lvs_name) + if leaders: + return "led", leaders, f"{lvs_name} led by {leaders[0].get_id()}" + if unknown: + return "skip", [], f"{lvs_name}: a member could not be asked for its leadership" + return "leaderless", members, f"{lvs_name}: no member of {len(members)} reports leadership" + + +def restore_healthy_lvs_leadership(owner_id, lvs_name=None, reason=""): + """Give a leader back to an LVS that is leaderless while all its members + are online and healthy (healthy_lvs_leadership_state); ``(leader, why)``. + + bf32 2026-11-21 (R6, target LVS_7 and LVS_10): with two of five nodes' + SPDK gone, a write to each LVS failed and its primary SPDK dropped the + leadership itself (spdk_lvs_queued_failed_IO, "Leadership state changed + internally to false"). It takes it back only on the next IO it receives, + and a replication target receives none -- replication waits for a leader + before it sends. The restarts that followed rebuilt the other members as + non-leaders "behind" the demoted primaries without asking whether they + led, and nothing else restores the leadership of a healthy LVS: both + stayed leaderless for 76 min with every member online. + + The grant is the guarded leaderless recovery the object paths use + (_recover_leaderless_lvs: single-flight lock with cooldown, no restart / + port-allow task on a member, JM quorum on the taker, every other member + proven non-leader twice, blob-md reload before set_leader, grant verified), + with the primary as the taker. Nothing is granted when any member is not + online or cannot be asked.""" + state, members, why = healthy_lvs_leadership_state(owner_id, lvs_name) + if state == "led": + return members[0], why + if state != "leaderless": + return None, why + owner = members[0] + logger.warning("LVS %s is leaderless with every member online (%s)%s -- restoring " + "leadership on its primary %s through the guarded leaderless recovery", + owner.lvstore, ", ".join(m.get_id()[:8] for m in members), + f" [{reason}]" if reason else "", owner.get_id()) + taker = _recover_leaderless_lvs(owner.cluster_id, members, owner.lvstore, owner) + if taker is None: + return None, f"{owner.lvstore}: guarded leaderless recovery granted nothing (see its log)" + return taker, f"{owner.lvstore}: leadership restored on {taker.get_id()}" + + def resync_solo_lvs_on_leaders(snode): """After a peer returns: start the journal resync of every LVS its surviving peer ran alone, on that LVS's LEADER only. diff --git a/simplyblock_core/test/conftest.py b/simplyblock_core/test/conftest.py index 773897acd2..d87cf7734c 100644 --- a/simplyblock_core/test/conftest.py +++ b/simplyblock_core/test/conftest.py @@ -84,3 +84,15 @@ def ns_host_guard_permits(monkeypatch): the runner tests that want a stranded host patch removal_blocked.""" from simplyblock_core.controllers import ns_host_guard monkeypatch.setattr(ns_host_guard, "removal_blocked", lambda *a, **kw: None) + + +@pytest.fixture(autouse=True) +def migration_space_permits(monkeypatch): + """Migration snapshots wait for free space and for the volume's pending + deletions (simplyblock_core/migration_space.py). The fakes here model no + capacity records and no snapshot table, so neither ever holds; + test_migration_space_bf29 exercises both with the real functions.""" + from simplyblock_core import migration_space + monkeypatch.setattr(migration_space, "space_hold", + lambda *a, **kw: ("", 0)) + monkeypatch.setattr(migration_space, "deletions_pending", lambda *a, **kw: []) diff --git a/simplyblock_core/test/test_cutover_target_down_bf29_r3b.py b/simplyblock_core/test/test_cutover_target_down_bf29_r3b.py new file mode 100644 index 0000000000..915f6f4896 --- /dev/null +++ b/simplyblock_core/test/test_cutover_target_down_bf29_r3b.py @@ -0,0 +1,170 @@ +"""An endgame whose target is down must not stop the volume's replication. + +Bed bf29 round R3b (2026-11-20): the target primary node was shut down +between replication cycles of an A -> B migration. One volume was in its +cutover endgame; the endgame waited for the target primary (the cutover +clone is built through it), and while it held the volume nothing replicated +it -- the endgame takes no snapshot while it waits, and the cadence snapshot +skips a volume in its endgame. No cycle completed for ~20 min, RPO 1280 s. + +After REPL_CUTOVER_TARGET_DOWN_GRACE_SEC the endgame is given up (no attempt +burned, no back-off): round state dropped, lvstore claim and cluster endgame +slot released, source write cap lifted. The volume replicates on its cadence +until the target is back, then enters a fresh endgame. +""" +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from simplyblock_core import constants +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_monitor +from simplyblock_core.services import tasks_runner_replication_final as runner +from simplyblock_core.services.task_runner_base import TaskDefer + +NOW = 2_000_000.0 +GRACE = constants.REPL_CUTOVER_TARGET_DOWN_GRACE_SEC + + +def _node(node_id, status): + return SimpleNamespace(get_id=lambda: node_id, status=status, cluster_id="A") + + +class _DB: + def __init__(self, tasks, tgt_status=StorageNode.STATUS_OFFLINE): + self.tasks = tasks + self.nodes = {"SRC": _node("SRC", StorageNode.STATUS_ONLINE), + "TGT": _node("TGT", tgt_status)} + self.lvol = MagicMock() + self.lvol.get_id.return_value = "LV1" + self.lvol.lvs_name = "LVS_1" + self.lvol.node_id = "SRC" + + def get_lvol_by_id(self, lvol_id): + return self.lvol + + def get_storage_node_by_id(self, node_id): + return self.nodes[node_id] + + def get_job_tasks(self, cluster_id, **kw): + return list(self.tasks) + + +def _endgame_task(**extra): + t = JobSchedule() + t.uuid = "T1" + t.cluster_id = "A" + t.function_name = JobSchedule.FN_REPLICATION_FINAL + t.status = JobSchedule.STATUS_RUNNING + t.retry = 0 + t.max_retry = 8 + t.function_params = {"lvol_id": "LV1", "src_node_id": "SRC", "tgt_node_id": "TGT", + "start_time": int(NOW - 3600), **extra} + return t + + +IN_ENDGAME = dict(cutover_lvs="LVS_1", cutover_cluster="A", cutover_claimed_at=NOW - 300, + shrink_snap_id="S5", shrink_round=5, shrink_round_times=[30.0] * 4, + shrink_started_at=NOW - 20, shrink_deadline=NOW + 600, + direct_rounds=False, throttle_restore=[0, 0, 0, 0]) + + +@pytest.fixture +def env(monkeypatch): + clock = {"t": NOW} + monkeypatch.setattr(runner.time, "time", lambda: clock["t"]) + spies = SimpleNamespace(throttle=MagicMock(), requeue=MagicMock()) + monkeypatch.setattr(runner, "_restore_source_throttle", spies.throttle) + monkeypatch.setattr(runner, "_requeue_direct_rounds", spies.requeue) + monkeypatch.setattr(runner, "_discard_spare_landing", lambda t: t) + + def setup(task, **kw): + fake = _DB([task], **kw) + monkeypatch.setattr(runner, "db", fake) + monkeypatch.setattr(snapshot_monitor, "db", fake) + return fake + return SimpleNamespace(clock=clock, spies=spies, setup=setup) + + +def _pass(task): + with pytest.raises(TaskDefer) as e: + runner.task_runner(task.frozen_view(), [task]) + return str(e.value) + + +def test_bf29_r3b_a_short_target_blip_keeps_the_endgame(env): + task = _endgame_task(**IN_ENDGAME) + env.setup(task) + assert "target node not online" in _pass(task) + assert task.function_params["target_down_since"] == NOW + env.clock["t"] = NOW + GRACE - 1 + _pass(task) + assert task.function_params["cutover_lvs"] == "LVS_1" + assert task.function_params["shrink_snap_id"] == "S5" + env.spies.throttle.assert_not_called() + + +def test_bf29_r3b_a_target_down_past_the_grace_returns_the_volume_to_its_cadence(env): + task = _endgame_task(**IN_ENDGAME) + env.setup(task) + _pass(task) + # the volume is held out of cadence while its endgame holds it ... + assert snapshot_monitor._cutover_endgame_lvol_ids("A") == {"LV1"} + env.clock["t"] = NOW + GRACE + _pass(task) + params = task.function_params + for key in ("cutover_lvs", "cutover_cluster", "cutover_claimed_at", "shrink_snap_id", + "shrink_round", "shrink_round_times"): + assert key not in params, key + # ... and is back on the ordinary cadence once the endgame is suspended + assert snapshot_monitor._cutover_endgame_lvol_ids("A") == set() + assert "endgame suspended" in params["last_error"] + assert task.retry == 0, "a target outage burns no attempt" + assert "endgame_retry_after" not in params, "no back-off: resume when the target is back" + env.spies.throttle.assert_called_once() + env.spies.requeue.assert_called_once() + + +def test_bf29_r3b_the_released_slot_is_free_for_another_volume(env, monkeypatch): + task = _endgame_task(**IN_ENDGAME) + env.setup(task) + _pass(task) + env.clock["t"] = NOW + GRACE + _pass(task) + other = _endgame_task() + other.uuid = "T2" + assert runner._cluster_endgame_slots_taken(other.frozen_view(), [task, other], "", "A") == "" + + +def test_bf29_r3b_a_volume_not_in_its_endgame_is_only_deferred(env): + task = _endgame_task(shrink_deadline=NOW + 900) + env.setup(task) + _pass(task) + env.clock["t"] = NOW + 10 * GRACE + _pass(task) + assert task.function_params["shrink_deadline"] == NOW + 900 + env.spies.throttle.assert_not_called() + env.spies.requeue.assert_not_called() + + +def test_bf29_r3b_the_target_back_clears_the_outage_and_the_endgame_can_resume(env, monkeypatch): + task = _endgame_task(**IN_ENDGAME) + fake = env.setup(task) + _pass(task) + env.clock["t"] = NOW + GRACE + _pass(task) + fake.nodes["TGT"].status = StorageNode.STATUS_ONLINE + + class _Stop(Exception): + pass + + def _lag(lvol): + raise _Stop() + monkeypatch.setattr(runner, "_replication_lag_sec", _lag) + with pytest.raises(_Stop): + # past the target check: the volume is on its way to the catch-up gate + # that admits it to a fresh endgame + runner.task_runner(task.frozen_view(), [task]) + assert "target_down_since" not in task.function_params diff --git a/simplyblock_core/test/test_migration_space_bf29.py b/simplyblock_core/test/test_migration_space_bf29.py new file mode 100644 index 0000000000..637ad7a6df --- /dev/null +++ b/simplyblock_core/test/test_migration_space_bf29.py @@ -0,0 +1,453 @@ +"""A migration must never fill its SOURCE cluster by itself (bed bf29). + +Bed bf29 (2026-11-20): two 5-node clusters, 8 client volumes under sustained +fio overwrite, a cluster-scope migration A -> B. Five volumes ran their +cutover endgame at once; each shrink round snapshotted its volume every +15-30 s, every snapshot pinned what fio overwrote after it, and the +superseded snapshots were deleted more slowly than new ones were taken. +Cluster A went 41% -> 82% -> 99% in 16 minutes and ran out of space. + +Three rules (simplyblock_core/migration_space.py): + +1. at most REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER endgames per source cluster; + the others keep replicating on their ordinary cadence; +2. no next round snapshot while the volume's earlier deletions are pending; +3. no migration snapshot (round or cadence) while the source's free space is + under the headroom estimate; an unreadable capacity holds only briefly. + +None of them touches client IO: only the migration's own snapshots wait. +""" +from types import SimpleNamespace +from typing import cast +from unittest.mock import MagicMock, patch + +import pytest + +from simplyblock_core import constants, migration_space +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_monitor +from simplyblock_core.services import tasks_runner_replication_final as runner +from simplyblock_core.services.task_runner_base import TaskDefer + +# The real functions: conftest neutralises both for every other test. +REAL_SPACE_HOLD = migration_space.space_hold +REAL_DELETIONS_PENDING = migration_space.deletions_pending + +GiB = 1 << 30 +NOW = 1_000_000.0 + + +@pytest.fixture +def real_rules(monkeypatch): + monkeypatch.setattr(migration_space, "space_hold", REAL_SPACE_HOLD) + monkeypatch.setattr(migration_space, "deletions_pending", REAL_DELETIONS_PENDING) + + +# --------------------------------------------------------------------------- # +# fakes +# --------------------------------------------------------------------------- # + +def _task(uuid, lvol_id, cluster="A", **params): + t = JobSchedule() + t.uuid = uuid + t.cluster_id = cluster + t.function_name = JobSchedule.FN_REPLICATION_FINAL + t.status = JobSchedule.STATUS_RUNNING + t.max_retry = 8 + t.date = 0 + t.function_params = {"lvol_id": lvol_id, **params} + return t + + +def _lvol(uuid, lvs, node="N-A", size=10 * GiB, mode="migration"): + lv = MagicMock() + lv.get_id.return_value = uuid + lv.uuid = uuid + lv.lvs_name = lvs + lv.node_id = node + lv.size = size + lv.top_bdev = f"LVS/{uuid}" + lv.replication_policy_id = "" + lv.replication_mode = mode + lv.pool_uuid = "P" + return lv + + +def _snap(snap_id, status=SnapShot.STATUS_ONLINE): + return SimpleNamespace(get_id=lambda: snap_id, status=status) + + +class _DB: + """The DB reads the rules make: nodes, tasks, capacity, snapshots.""" + + def __init__(self, tasks=(), free=None, total=1000 * GiB, record_age=5.0, + snaps=None, dirty=None, nodes=None): + self.tasks = list(tasks) + self.total = total + self.free = free + self.record_age = record_age + self.snaps = snaps or {} + self.dirty = dirty + self.rpc_calls = 0 + self.now = NOW # when the latest record was written + age + self.nodes = nodes or {"N-A": self._node("N-A", "A"), "N-B": self._node("N-B", "B")} + + def _node(self, node_id, cluster): + def info(bdev): + self.rpc_calls += 1 + if self.dirty is None: + raise RuntimeError("no count") + return {"tracking": True, "complete": True, "dirty_bytes": self.dirty} + rpc = SimpleNamespace(bdev_lvol_dirty_bitmap_info=info) + return SimpleNamespace(get_id=lambda: node_id, cluster_id=cluster, + status=StorageNode.STATUS_ONLINE, + secondary_node_id="", tertiary_node_id="", + rpc_client=lambda **kw: rpc) + + def get_storage_node_by_id(self, node_id): + return self.nodes[node_id] + + def get_job_tasks(self, cluster_id, **kw): + return [t for t in self.tasks if t.cluster_id == cluster_id] + + def get_cluster_by_id(self, cluster_id): + return SimpleNamespace(get_id=lambda: cluster_id) + + def get_cluster_capacity(self, cluster, limit=1): + if self.free is None: + return [] + return [SimpleNamespace(size_total=self.total, size_used=self.total - self.free, + date=int(self.now - self.record_age))] + + def get_snapshots_by_lvol_id(self, lvol_id): + return list(self.snaps.get(lvol_id, ())) + + +@pytest.fixture +def clock(monkeypatch): + now = {"t": NOW} + monkeypatch.setattr(runner.time, "time", lambda: now["t"]) + return now + + +def _use(monkeypatch, fake): + monkeypatch.setattr(runner, "db", fake) + monkeypatch.setattr(snapshot_monitor, "db", fake) + + +# --------------------------------------------------------------------------- # +# Rule 1: one endgame per source cluster +# --------------------------------------------------------------------------- # + +def test_bf29_a_second_lvstore_of_the_same_cluster_queues_for_the_endgame_slot(monkeypatch, clock): + """bf29: five endgames on five lvstores of cluster A ran at once. The + per-lvstore claim let them; the cluster slot does not.""" + holder = _task("T1", "LV1", cutover_lvs="LVS_1", cutover_claimed_at=NOW - 60) + mine = _task("T2", "LV2") + _use(monkeypatch, _DB([holder, mine])) + with pytest.raises(TaskDefer, match="endgame slot of source cluster A"): + runner._acquire_lvs_claim(mine.frozen_view(), _lvol("LV2", "LVS_2"), [holder, mine]) + assert "cutover_lvs" not in mine.function_params, "a queued volume holds no lvstore" + + +def test_bf29_the_queued_volume_keeps_its_ordinary_cadence(monkeypatch): + """Waiting for the slot is not an endgame: the cadence snapshots of the + queued volume go on (no RPO gap added by the wait).""" + holder = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_snap_id="S1") + queued = _task("T2", "LV2", shrink_deadline=NOW + 900) + _use(monkeypatch, _DB([holder, queued])) + assert snapshot_monitor._cutover_endgame_lvol_ids("A") == {"LV1"} + + +def test_bf29_an_endgame_of_another_cluster_does_not_take_the_slot(monkeypatch, clock): + other = _task("T1", "LVB", cluster="B", cutover_lvs="LVS_1", cutover_cluster="B") + mine = _task("T2", "LV2") + _use(monkeypatch, _DB([other, mine])) + out = runner._acquire_lvs_claim(mine.frozen_view(), _lvol("LV2", "LVS_2"), [other, mine]) + assert out.function_params["cutover_lvs"] == "LVS_2" + assert out.function_params["cutover_cluster"] == "A" + + +def test_bf29_a_consistency_group_sibling_joins_its_groups_slot(monkeypatch, clock): + holder = _task("T1", "LV1", cutover_lvs="LVS_1", cutover_group="G1", + cutover_claimed_at=NOW - 60) + mine = _task("T2", "LV2") + _use(monkeypatch, _DB([holder, mine])) + monkeypatch.setattr(runner, "_group_id_for_lvol", lambda lv: "G1") + out = runner._acquire_lvs_claim(mine.frozen_view(), _lvol("LV2", "LVS_1"), [holder, mine]) + assert out.function_params["cutover_lvs"] == "LVS_1" + + +def test_bf29_the_limit_is_configurable(monkeypatch, clock): + monkeypatch.setattr(constants, "REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER", 2) + holder = _task("T1", "LV1", cutover_lvs="LVS_1", cutover_claimed_at=NOW - 60) + second = _task("T2", "LV2") + third = _task("T3", "LV3") + fake = _DB([holder, second, third]) + _use(monkeypatch, fake) + runner._acquire_lvs_claim(second.frozen_view(), _lvol("LV2", "LVS_2"), [holder, second, third]) + assert second.function_params["cutover_lvs"] == "LVS_2" + with pytest.raises(TaskDefer, match="2 of 2 in use"): + runner._acquire_lvs_claim(third.frozen_view(), _lvol("LV3", "LVS_3"), + [holder, second, third]) + + +def test_bf29_claims_racing_in_one_pass_leave_exactly_one_endgame(monkeypatch, clock): + """All five caught up in the same runner pass: the pass's task list shows + no claim to any of them. The claim is re-checked against a fresh read + after it is written, and only the first claimant keeps it.""" + tasks = [_task(f"T{i}", f"LV{i}") for i in range(1, 6)] + stale = [JobSchedule().from_dict(t.to_dict()) for t in tasks] # the pass's view + _use(monkeypatch, _DB(tasks)) + winners, queued = [], [] + for i, t in enumerate(tasks, start=1): + clock["t"] = NOW + i + try: + runner._acquire_lvs_claim(t.frozen_view(), _lvol(f"LV{i}", f"LVS_{i}"), stale) + winners.append(t.uuid) + except TaskDefer as e: + assert "lost to a concurrent claim" in str(e) + queued.append(t.uuid) + assert winners == ["T1"] + assert queued == ["T2", "T3", "T4", "T5"] + for t in tasks[1:]: + assert not t.function_params.get("cutover_lvs"), "a loser withdraws its claim" + + +def test_bf29_a_later_claim_never_displaces_a_running_endgame(monkeypatch, clock): + """Ranking is by claim time, not task age: an older task that claims + late must not take the slot from the endgame already running.""" + running = _task("T9", "LV9", cutover_lvs="LVS_9", cutover_claimed_at=NOW - 300) + older = _task("T0", "LV0") # sorts first by id/creation + _use(monkeypatch, _DB([running, older])) + with pytest.raises(TaskDefer): + runner._confirm_cluster_endgame_slot( + runner._record(older.frozen_view(), cutover_lvs="LVS_0", + cutover_cluster="A", cutover_claimed_at=NOW), "", "A") + assert not older.function_params.get("cutover_lvs") + + +def test_bf29_releasing_the_claim_frees_the_slot(monkeypatch, clock): + holder = _task("T1", "LV1", cutover_lvs="LVS_1", cutover_cluster="A", + cutover_claimed_at=NOW - 60) + _use(monkeypatch, _DB([holder])) + monkeypatch.setattr(runner, "_requeue_direct_rounds", lambda t: None) + runner._release_lvs_claim(holder.frozen_view()) + for key in ("cutover_lvs", "cutover_cluster", "cutover_claimed_at"): + assert key not in holder.function_params + + +# --------------------------------------------------------------------------- # +# Rule 2: the next round waits for the volume's pending deletions +# --------------------------------------------------------------------------- # + +def _hold_db(**kw): + kw.setdefault("free", 900 * GiB) + return _DB(**kw) + + +def test_bf29_no_round_snapshot_while_an_earlier_one_is_still_being_deleted( + monkeypatch, clock, real_rules): + fake = _hold_db(snaps={"LV1": [_snap("S1", SnapShot.STATUS_IN_DELETION), + _snap("S2")]}) + _use(monkeypatch, fake) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=3, direct_rounds=False) + taken = MagicMock() + monkeypatch.setattr(runner, "_take_shrink_snapshot", taken) + err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert "round 4 snapshot waits for 1 deletion" in err + taken.assert_not_called() + assert task.function_params["deletion_wait_since"] == NOW + + +def test_bf29_the_round_is_taken_once_the_deletion_completed(monkeypatch, clock, real_rules): + fake = _hold_db(snaps={"LV1": [_snap("S2")]}) + _use(monkeypatch, fake) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=3, direct_rounds=False, + deletion_wait_since=NOW - 20) + taken = MagicMock(return_value=("S3", None, task)) + monkeypatch.setattr(runner, "_take_shrink_snapshot", lambda t, lv, direct=False: taken(t)) + err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert err is None + taken.assert_called_once() + assert "deletion_wait_since" not in task.function_params + + +def test_bf29_a_stuck_deletion_is_set_aside_after_the_bounded_wait(monkeypatch, clock, real_rules): + """A deletion that never completes (no leader, a clone in deletion) must + not stall the endgame on its own; a newer pending deletion is still + waited for.""" + fake = _hold_db(snaps={"LV1": [_snap("STUCK", SnapShot.STATUS_IN_DELETION)]}) + _use(monkeypatch, fake) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=3, direct_rounds=False, + deletion_wait_since=NOW - constants.REPL_ENDGAME_DELETION_WAIT_MAX_SEC) + why, _ = runner._round_snapshot_hold(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert why == "" + assert task.function_params["deletion_ignored"] == ["STUCK"] + fake.snaps["LV1"].append(_snap("S4", SnapShot.STATUS_IN_DELETION)) + why, _ = runner._round_snapshot_hold(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert "S4" in why and "STUCK" not in why + + +def test_bf29_a_soft_deleted_snapshot_is_not_a_pending_deletion(monkeypatch, real_rules): + soft = SimpleNamespace(get_id=lambda: "S1", status=SnapShot.STATUS_ONLINE, deleted=True) + assert migration_space.deletions_pending(_DB(snaps={"LV1": [soft]}), _lvol("LV1", "L")) == [] + + +def test_bf29_a_held_round_is_not_measured_twice(monkeypatch, clock, real_rules): + """The round finished; the next snapshot waits for a deletion. The next + pass must not book the waiting time as another, longer round.""" + fake = _hold_db(snaps={"LV1": [_snap("S0", SnapShot.STATUS_IN_DELETION)]}) + _use(monkeypatch, fake) + monkeypatch.setattr(runner, "_shrink_round_done", lambda snap_id: True) + monkeypatch.setattr(runner, "_take_shrink_snapshot", MagicMock()) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=2, shrink_snap_id="S1", + shrink_started_at=NOW - 10, shrink_deadline=NOW + 900, + shrink_round_times=[20.0], direct_rounds=False) + lvol = _lvol("LV1", "LVS_1") + with patch.object(runner, "_throttle_source", lambda t, lv: t): + with pytest.raises(TaskDefer, match="waits for 1 deletion"): + runner._shrink_step(task.frozen_view(), lvol) + assert task.function_params["shrink_round_times"] == [20.0, 10.0] + clock["t"] += 60 + with pytest.raises(TaskDefer, match="waits for 1 deletion"): + runner._shrink_step(task.frozen_view(), lvol) + assert task.function_params["shrink_round_times"] == [20.0, 10.0] + cast(MagicMock, runner._take_shrink_snapshot).assert_not_called() + + +def test_bf29_the_endgame_is_not_entered_while_its_first_round_would_wait( + monkeypatch, clock, real_rules): + """Holding the lvstore and the cluster slot through the wait would stop + the other volumes' transfers for nothing.""" + fake = _hold_db(snaps={"LV1": [_snap("S0", SnapShot.STATUS_IN_DELETION)]}) + _use(monkeypatch, fake) + held, _ = runner._round_snapshot_hold(_task("T1", "LV1").frozen_view(), _lvol("LV1", "L")) + assert held + import inspect + src = inspect.getsource(runner.task_runner) + assert src.index("_round_snapshot_hold") < src.index("_acquire_lvs_claim") + + +# --------------------------------------------------------------------------- # +# Rule 3: free space on the source cluster +# --------------------------------------------------------------------------- # + +def test_bf29_no_round_snapshot_with_the_source_nearly_full(monkeypatch, clock, real_rules): + """82% -> 99%: under the 5% floor no migration snapshot is taken, and an + event says why -- once per wait, not once per pass.""" + fake = _hold_db(free=10 * GiB) # 99% used of 1000 GiB + _use(monkeypatch, fake) + events = [] + monkeypatch.setattr(migration_space, "space_low_event", + lambda lv, why: events.append(why)) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=5, direct_rounds=False) + monkeypatch.setattr(runner, "_take_shrink_snapshot", MagicMock()) + err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert "deferred" in err and "floor" in err + err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert err + assert len(events) == 1 + cast(MagicMock, runner._take_shrink_snapshot).assert_not_called() + assert fake.rpc_calls == 0, "under the floor there is nothing to ask SPDK" + + +def test_bf29_space_recovered_lets_the_round_go(monkeypatch, clock, real_rules): + fake = _hold_db(free=900 * GiB) + _use(monkeypatch, fake) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=5, direct_rounds=False, + space_low_since=NOW - 120) + monkeypatch.setattr(runner, "_take_shrink_snapshot", + lambda t, lv, direct=False: ("S6", None, t)) + err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert err is None + assert "space_low_since" not in task.function_params + + +def test_bf29_headroom_is_twice_the_bytes_written_since_the_last_snapshot(monkeypatch): + lvol = _lvol("LV1", "L", size=400 * GiB) + # 100 GiB free (> 5% floor of 50 GiB); 60 GiB dirty needs 120 GiB + fake = _DB(free=100 * GiB, dirty=60 * GiB) + verdict, why = migration_space.assess(fake, lvol, "A", now=NOW) + assert verdict == "low" and "120 GiB headroom" in why + fake.dirty = 30 * GiB # needs 60 GiB: fits + assert migration_space.assess(fake, lvol, "A", now=NOW)[0] == "ok" + + +def test_bf29_a_small_volume_on_a_roomy_cluster_asks_spdk_nothing(monkeypatch): + fake = _DB(free=500 * GiB, dirty=1 * GiB) + assert migration_space.assess(fake, _lvol("LV1", "L", size=10 * GiB), "A", now=NOW)[0] == "ok" + assert fake.rpc_calls == 0 + + +def test_bf29_an_unknown_dirty_count_assumes_the_whole_volume(monkeypatch): + fake = _DB(free=100 * GiB, dirty=None) # RPC fails + verdict, _ = migration_space.assess(fake, _lvol("LV1", "L", size=60 * GiB), "A", now=NOW) + assert verdict == "low" # 2 x 60 GiB > 100 GiB + + +def test_bf29_unreadable_capacity_holds_only_for_a_bounded_time(monkeypatch, real_rules): + """Fail-safe: no capacity record (collector down) holds the snapshot for + REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC, then lets it through.""" + fake = _DB(free=None) + lvol = _lvol("LV1", "L") + why, since = migration_space.space_hold(fake, lvol, "A", 0, now=NOW) + assert why and since == NOW + why, since = migration_space.space_hold(fake, lvol, "A", since, now=NOW + 60) + assert why and since == NOW + why, since = migration_space.space_hold( + fake, lvol, "A", since, now=NOW + constants.REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC) + assert why == "" and since == NOW + + +def test_bf29_a_stale_capacity_record_is_no_reading(monkeypatch): + fake = _DB(free=900 * GiB, record_age=constants.REPL_MIGRATION_CAPACITY_STALE_SEC + 1) + verdict, why = migration_space.assess(fake, _lvol("LV1", "L"), "A", now=NOW) + assert verdict == "unknown" and "old" in why + + +def test_bf29_known_low_space_is_never_waited_out(monkeypatch, real_rules): + """Only an UNREADABLE capacity is bounded; a known-full source keeps the + migration snapshot waiting (the endgame's shrink deadline bounds that).""" + fake = _DB(free=1 * GiB) + why, since = migration_space.space_hold(fake, _lvol("LV1", "L"), "A", NOW - 10 ** 6, + now=NOW) + assert why and since == 0 + + +def test_bf29_cadence_of_a_migrating_volume_waits_for_space(monkeypatch, real_rules): + fake = _DB(free=10 * GiB) + _use(monkeypatch, fake) + events = [] + monkeypatch.setattr(migration_space, "space_low_event", + lambda lv, why: events.append(why)) + snapshot_monitor._space_low.clear() + snapshot_monitor._space_unknown_since.clear() + lvol = _lvol("LV1", "L", mode="migration") + assert snapshot_monitor._migration_space_hold(lvol, "A", NOW) + assert snapshot_monitor._migration_space_hold(lvol, "A", NOW + 60) + assert len(events) == 1 + fake.free = 900 * GiB + fake.now = NOW + 120 + assert snapshot_monitor._migration_space_hold(lvol, "A", NOW + 120) == "" + assert "LV1" not in snapshot_monitor._space_low + + +def test_bf29_dr_replication_cadence_is_never_held_for_space(monkeypatch, real_rules): + """The rules bound a migration's own consumption only: a volume that + replicates for DR is not a migration and keeps its cadence.""" + fake = _DB(free=1 * GiB) + _use(monkeypatch, fake) + lvol = _lvol("LV1", "L", mode="replicate") + assert snapshot_monitor._migration_space_hold(lvol, "A", NOW) == "" + + +def test_bf29_client_volume_creation_does_not_consult_the_migration_rules(): + """Capacity thresholds stay alarms: nothing on the client path (volume + create, IO) imports the migration space rules.""" + import inspect + + from simplyblock_core.controllers import lvol_controller + assert "migration_space" not in inspect.getsource(lvol_controller) diff --git a/simplyblock_core/test/test_ns_host_guard.py b/simplyblock_core/test/test_ns_host_guard.py index 3da5df761d..728894618f 100644 --- a/simplyblock_core/test/test_ns_host_guard.py +++ b/simplyblock_core/test/test_ns_host_guard.py @@ -59,6 +59,25 @@ def test_bed_bf26_a_host_only_on_the_side_that_goes_is_stranded(guard): assert H1 in why and "discarding clone C" in why +def test_bed_bf28_one_machine_under_two_host_nqns_is_not_stranded(guard): + # Source paths carry the simplyblock host NQN, the early-prepared target + # paths the kernel default: the same client, never stranded. + sb = "nqn.2014-08.io.simplyblock:uuid:5892b49a-bc39-41b2-a38a-8daae13a9d0d" + kernel = "nqn.2014-08.org.nvmexpress:uuid:ec2d55d6-1c18-ced7-92ee-34f409cb58ce" + db = _DB(_node("S1", [sb]), _node("S2", [sb]), _node("T1", [kernel]), _node("T2", [kernel])) + # the cutover freeze: the source paths go inaccessible, the target ones stay + assert guard.stranded_hosts(NQN, ["S1", "S2"], ["T1", "T2"], db=db) == set() + # a discarded target clone: the target paths go, the source ones stay + assert guard.removal_blocked(NQN, ["T1", "T2"], ["S1", "S2"], "x", db=db) is None + + +def test_more_unmatched_hosts_going_than_staying_are_stranded(guard): + # Two clients on the side that goes, one other name on the side that stays: + # at least one machine loses its last path; both candidates are named. + db = _DB(_node("T1", [H1, H2]), _node("S1", ["nqn.host:other"])) + assert guard.stranded_hosts(NQN, ["T1"], ["S1"], db=db) == {H1, H2} + + def test_nodes_that_are_down_or_lack_the_subsystem_serve_no_host(guard): db = _DB(_node("T1", [H1]), _node("S1", [H1], status=StorageNode.STATUS_OFFLINE), @@ -127,3 +146,25 @@ def blocked(nqn, removing, keeping, what, db=None): src = inspect.getsource(runner) i = src.index("cutover freeze of") assert i < src.index("freeze_started_at=time.time()") + + +def test_bed_bf29_the_source_delete_finds_the_target_by_nqn_after_the_clone_is_gone(monkeypatch): + # After the cutover the clone named in tgt_lvol_id no longer exists; the + # volume runs on the other cluster under a new record with the same NQN. + src = SimpleNamespace(get_id=lambda: "SRC", nqn=NQN, node_id="S1", nodes=["S2"], + status="online") + live = SimpleNamespace(get_id=lambda: "NEW", nqn=NQN, node_id="T1", nodes=["T2", "T3"], + status="online") + other = SimpleNamespace(get_id=lambda: "X", nqn="nqn.other", node_id="T9", nodes=[], + status="online") + going = SimpleNamespace(get_id=lambda: "OLD", nqn=NQN, node_id="T7", nodes=[], + status="in_deletion") + vols = {"A": [src], "B": [live, other, going]} + + def get_lvol_by_id(i): + raise KeyError(i) + monkeypatch.setattr(runner, "db", SimpleNamespace( + get_lvol_by_id=get_lvol_by_id, + get_clusters=lambda: [SimpleNamespace(get_id=lambda: "A"), SimpleNamespace(get_id=lambda: "B")], + get_lvols=lambda cid: vols[cid])) + assert runner._live_target_node_ids(src, {"tgt_lvol_id": "GONE"}) == ["T1", "T2", "T3"] diff --git a/simplyblock_core/test/test_repl_double_run_bf32.py b/simplyblock_core/test/test_repl_double_run_bf32.py new file mode 100644 index 0000000000..7c09b0a495 --- /dev/null +++ b/simplyblock_core/test/test_repl_double_run_bf32.py @@ -0,0 +1,386 @@ +"""One replication task per snapshot, run by one worker; a half-created +landing volume is deleted only when provably abandoned, and everywhere. + +bed bf32 (2026-11-21, cluster 70edb423): two replication workers enqueued the +same chain ancestor (snapshot 4d20399d) 100 ms apart. Each scanned the task +table, found no task, and wrote one: 5571400c and a6f9b7b7. Both ran on the +worker pool at once -- the pool dedupes by task uuid only, and _dispatch only +yielded to a sibling that was already RUNNING. 5571400c created landing +volume 5eeb989b (REP_repl_internal_7438cbe8_1795231066); 2 s into that create +a6f9b7b7 found the record in_creation, took it for a dead attempt's leftover +and deleted it. The delete ran while the creator was still registering +LVOL_2398 on the secondary and tertiary, so those kept a registration nothing +tracked; SPDK reused the blob id and both nodes crashed 15 min later. +""" +import threading +import time +from datetime import datetime, timedelta +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from simplyblock_core import constants +from simplyblock_core.controllers import tasks_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_replication as sr + +SNAP = "4d20399d-acad-4a11-a1e1-7d8ebd9a44d8" +REP_NAME = "REP_repl_internal_7438cbe8_1795231066" + + +class FakeKV: + """Dict-backed store for the transactionless path (see test_ns_slot_txn).""" + + def __init__(self): + self.data = {} + + def get(self, key): + return self.data.get(key) + + def set(self, key, value): + self.data[key] = value + + def clear(self, key): + self.data.pop(key, None) + + def add(self, key, value): + pass + + def get_range_startswith(self, prefix, limit=0, reverse=False): + return sorted((k, v) for k, v in self.data.items() if k.startswith(prefix)) + + +@pytest.fixture +def dbc(): + d = DBController() + saved = d.kv_store + d.kv_store = FakeKV() + yield d + d.kv_store = saved + + +def _repl_task(uuid, snap=SNAP, status=JobSchedule.STATUS_NEW, date=1795237024, **params): + t = JobSchedule() + t.uuid = uuid + t.cluster_id = "70edb423" + t.node_id = "node" + t.date = date + t.function_name = JobSchedule.FN_SNAPSHOT_REPLICATION + t.function_params = {"snapshot_id": snap, "replicate_to_source": True, **params} + t.status = status + t.max_retry = 8 + return t + + +# -- (1) one task per snapshot: the atomic claim -------------------------------- + + +def _recorder(ran): + """A task_runner stand-in that records which task ran.""" + def run(task): + ran.append(task.uuid) + return True + return run + + +def _snapshot_creating(seen): + """A create stand-in that records the in-flight registry while it runs.""" + def create(*_a): + seen.append(dict(sr._LANDING_CREATING)) + return ("id", None) + return create + +def test_bf32_second_task_of_a_snapshot_is_refused_by_the_claim(dbc): + first, created = dbc.ensure_snapshot_replication_task(_repl_task("5571400c")) + assert (first, created) == ("5571400c", True) + other, created = dbc.ensure_snapshot_replication_task(_repl_task("a6f9b7b7")) + assert (other, created) == ("5571400c", False) + assert dbc.get_task_by_id("5571400c").uuid == "5571400c" + with pytest.raises(KeyError): + dbc.get_task_by_id("a6f9b7b7") + + +def test_bf32_a_done_task_frees_the_snapshot_claim(dbc): + t = _repl_task("5571400c") + dbc.ensure_snapshot_replication_task(t) + t.status = JobSchedule.STATUS_DONE + t.write_to_db(dbc.kv_store) + assert dbc.ensure_snapshot_replication_task(_repl_task("a6f9b7b7")) == ("a6f9b7b7", True) + + +def test_bf32_two_concurrent_enqueues_of_one_ancestor_create_one_task(dbc): + """The bf32 interleaving: both enqueuers' scans find nothing. The write + is the claim (one FDB transaction in production; serialized here the way + FDB's conflict retry serializes it), so only one task is created.""" + txn = threading.Lock() + real = dbc.ensure_snapshot_replication_task + + def serialized(task): + with txn: + return real(task) + + stub = MagicMock() + stub.get_job_tasks.return_value = [] # both scans: no task yet + stub.ensure_snapshot_replication_task.side_effect = serialized + barrier = threading.Barrier(2) + results = [] + + def enqueue(): + barrier.wait() + results.append(tasks_controller.add_snapshot_replication_task( + "70edb423", "node", SNAP, replicate_to_source=True)) + + with patch.object(tasks_controller, "db", stub): + threads = [threading.Thread(target=enqueue) for _ in range(2)] + for th in threads: + th.start() + for th in threads: + th.join(5) + created = [r for r in results if r] + assert len(results) == 2 and len(created) == 1, results + stub.kv_store.assert_not_called() + assert sum(1 for k in dbc.kv_store.data if k.startswith(b"repl_snap_task/")) == 1 + + +# -- (1) one run per snapshot: the worker pool and _dispatch ---------------------- + +def _wait_until(cond, timeout=5.0): + end = time.time() + timeout + while time.time() < end: + if cond(): + return True + time.sleep(0.01) + return cond() + + +def test_bf32_two_tasks_of_one_snapshot_never_run_at_once_on_the_pool(): + runs, release = [], threading.Event() + + def slow(task): + runs.append(task.uuid) + release.wait(5) + return True + + pool = sr._Workers(4) + with patch.object(sr, "_dispatch", side_effect=slow): + pool.submit(_repl_task("5571400c")) + assert _wait_until(lambda: runs == ["5571400c"]) + pool.submit(_repl_task("a6f9b7b7")) # same snapshot, other task + time.sleep(0.1) + assert runs == ["5571400c"] + release.set() + assert _wait_until(lambda: pool.running() == 0) + pool.submit(_repl_task("a6f9b7b7")) # the first is done: may run now + assert _wait_until(lambda: runs == ["5571400c", "a6f9b7b7"]) + + +class _TaskDB: + def __init__(self, tasks): + self.tasks = {t.uuid: t for t in tasks} + + def get_replication_tasks_for_snapshot(self, snapshot_id): + return [t for t in self.tasks.values() + if t.function_params.get("snapshot_id") == snapshot_id] + + def get_task_by_id(self, uuid): + return self.tasks[uuid] + + +@pytest.mark.parametrize("status", [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED]) +def test_bf32_only_one_of_two_live_tasks_of_a_snapshot_is_dispatched(status): + """Neither task was RUNNING in bf32 (the landing create comes first), so + the old RUNNING-only yield let both through.""" + older = _repl_task("5571400c", status=status, date=100) + newer = _repl_task("a6f9b7b7", status=status, date=101) + ran: list[str] = [] + with patch.object(sr, "db", _TaskDB([older, newer])), \ + patch.object(sr, "task_runner", side_effect=_recorder(ran)): + assert sr._dispatch(newer) is True + assert sr._dispatch(older) is True + assert ran == ["5571400c"] + + +def test_bf32_the_task_owning_a_landing_volume_runs_first(): + older = _repl_task("5571400c", date=100) + newer = _repl_task("a6f9b7b7", date=101, remote_lvol_id="5eeb989b") + ran: list[str] = [] + with patch.object(sr, "db", _TaskDB([older, newer])), \ + patch.object(sr, "task_runner", side_effect=_recorder(ran)): + sr._dispatch(older) + sr._dispatch(newer) + assert ran == ["a6f9b7b7"] + + +# -- (2) the half-created landing cleanup ---------------------------------------- + +def _landing(age_sec, status=LVol.STATUS_IN_CREATION): + lv = LVol() + lv.uuid = "5eeb989b-5ca6-4ac0-9f44-4cacde14dbb3" + lv.lvol_name = REP_NAME + lv.status = status + lv.node_id = "9154330e" + lv.nodes = ["9154330e"] # the secondaries not recorded yet + lv.lvs_name = "LVS_10" + lv.lvol_bdev = "LVOL_2398" + lv.create_dt = str(datetime.now() - timedelta(seconds=age_sec)) + return lv + + +def _node(nid, status=StorageNode.STATUS_ONLINE, rpc=None, **kw): + n = SimpleNamespace(cluster_id="70edb423", status=status, + secondary_node_id="", tertiary_node_id="", **kw) + n.get_id = lambda: nid + n.rpc_client = lambda **_: rpc + return n + + +class _StartDB: + """Enough of the DB for process_snap_replicate_start to reach the + landing-volume probe.""" + + def __init__(self, landing, tasks, nodes): + self.landing, self.tasks, self.nodes = landing, tasks, nodes + self.deleted = False + + def get_storage_node_by_id(self, nid): + if nid not in self.nodes: + raise KeyError(nid) + return self.nodes[nid] + + def get_lvol_by_name(self, name, include_deleted=False): + if name != REP_NAME or self.deleted: + raise KeyError(name) + return self.landing + + def get_lvol_by_id(self, lvol_id): + if self.deleted or lvol_id != self.landing.get_id(): + raise KeyError(lvol_id) + return self.landing + + def get_replication_tasks_for_snapshot(self, snapshot_id): + return [t for t in self.tasks if t.function_params.get("snapshot_id") == snapshot_id] + + +def _run_start(task, sdb): + snapshot = SimpleNamespace( + snap_name="repl_internal_7438cbe8_1795231066", source_replicated_snap_uuid="", + target_replicated_snap_uuid="", cluster_id="70edb423", size=10 << 30, + lvol=SimpleNamespace(node_id="SRC", get_id=lambda: "7438cbe8"), + get_id=lambda: SNAP) + delete = MagicMock(side_effect=lambda lv, **kw: setattr(sdb, "deleted", True)) + suspended = [] + with patch.object(sr, "db", sdb), \ + patch.object(sr, "_lvs_transfer_hold", return_value=None), \ + patch.object(sr, "_merge_behind_hold", return_value=None), \ + patch.object(sr, "_select_source_node", + return_value=(SimpleNamespace(get_id=lambda: "SRC", cluster_id="SRCCL"), + "primary")), \ + patch.object(sr, "_unreplicated_local_ancestor", return_value=("ok", None, "")), \ + patch.object(sr, "_destination_pool_uuid", return_value="POOL"), \ + patch.object(sr, "_counterpart_on_destination", return_value=None), \ + patch.object(sr, "_create_landing_volume") as create, \ + patch.object(sr, "_suspend_for_retry", + side_effect=lambda t, msg, **kw: suspended.append(msg)), \ + patch.object(sr.lvol_controller, "delete_lvol", delete), patch.object(JobSchedule, "write_to_db"), \ + patch("simplyblock_core.controllers.tasks_controller.add_lvol_sync_del_task") as queued: + sr.process_snap_replicate_start(task, snapshot) + create.assert_not_called() + return delete, suspended, queued + + +def test_bf32_cleanup_does_not_delete_a_landing_volume_another_task_creates(): + """The exact bf32 state: a6f9b7b7 meets 5eeb989b in_creation, 2 s old, + while 5571400c (live) creates it.""" + me = _repl_task("a6f9b7b7", status=JobSchedule.STATUS_SUSPENDED) + creator = _repl_task("5571400c") + sdb = _StartDB(_landing(age_sec=2), [me, creator], + {"node": _node("node"), "9154330e": _node("9154330e")}) + delete, suspended, _ = _run_start(me, sdb) + delete.assert_not_called() + assert "5571400c" in suspended[0] and "under creation" in suspended[0] + + +def test_bf32_cleanup_does_not_delete_a_landing_volume_this_runner_creates(): + me = _repl_task("a6f9b7b7", status=JobSchedule.STATUS_SUSPENDED) + sdb = _StartDB(_landing(age_sec=constants.REPL_LANDING_ABANDON_SEC * 2), [me], + {"node": _node("node"), "9154330e": _node("9154330e")}) + with sr._LANDING_CREATING_GUARD: + sr._LANDING_CREATING[REP_NAME] = 1 + try: + delete, suspended, _ = _run_start(me, sdb) + finally: + with sr._LANDING_CREATING_GUARD: + sr._LANDING_CREATING.pop(REP_NAME, None) + delete.assert_not_called() + assert "running in this runner" in suspended[0] + + +def test_bf32_cleanup_does_not_delete_a_young_landing_volume(): + """No live sibling and nothing here creating it -- but a create in another + process (a runner pod being replaced) cannot be seen: age decides.""" + me = _repl_task("a6f9b7b7", status=JobSchedule.STATUS_SUSPENDED) + sdb = _StartDB(_landing(age_sec=30), [me], + {"node": _node("node"), "9154330e": _node("9154330e")}) + delete, suspended, _ = _run_start(me, sdb) + delete.assert_not_called() + assert "old" in suspended[0] + + +def test_bf32_a_landing_volume_mid_create_is_registered_while_add_lvol_ha_runs(): + seen: list[dict] = [] + with patch.object(sr, "_create_landing_volume_unguarded", + side_effect=_snapshot_creating(seen)): + assert sr._create_landing_volume(REP_NAME, None, None, None) == ("id", None) + assert seen == [{REP_NAME: 1}] + assert REP_NAME not in sr._LANDING_CREATING + + +class _Rpc: + def __init__(self, present): + self.present = present + self.deleted = [] + + def bdev_get(self, name): + return {"name": name} if self.present else None + + def delete_lvol(self, name, sync=False): + self.deleted.append((name, sync)) + self.present = False + return True + + +def test_bf32_cleanup_deletes_an_abandoned_landing_volume_on_the_secondaries_too(): + """Abandoned: deleted -- and confirmed gone on every lvstore member, even + the ones the record did not list yet. The secondary that kept LVOL_2398 + registered gets a sync delete; an offline tertiary a durable task.""" + me = _repl_task("a6f9b7b7", status=JobSchedule.STATUS_SUSPENDED) + prim_rpc, sec_rpc = _Rpc(False), _Rpc(True) + nodes = { + "node": _node("node"), + "9154330e": _node("9154330e", rpc=prim_rpc), + "8de868b9": _node("8de868b9", rpc=sec_rpc), + "2aa1550c": _node("2aa1550c", status=StorageNode.STATUS_OFFLINE), + } + nodes["9154330e"].secondary_node_id = "8de868b9" + nodes["9154330e"].tertiary_node_id = "2aa1550c" + sdb = _StartDB(_landing(age_sec=constants.REPL_LANDING_ABANDON_SEC + 60), [me], nodes) + delete, suspended, queued = _run_start(me, sdb) + delete.assert_called_once() + assert delete.call_args.kwargs == {"force_delete": True} + assert prim_rpc.deleted == [] + assert sec_rpc.deleted == [("LVS_10/LVOL_2398", True)] + queued.assert_called_once_with("70edb423", "2aa1550c", "LVS_10/LVOL_2398", "9154330e") + assert "cleared stale landing volume" in suspended[0] + + +def test_bf32_a_record_still_in_deletion_is_left_to_the_monitor(): + lv = _landing(age_sec=9999, status=LVol.STATUS_IN_DELETION) + rpc = _Rpc(True) + sdb = _StartDB(lv, [], {"9154330e": _node("9154330e", rpc=rpc)}) + with patch.object(sr, "db", sdb): + assert sr._confirm_landing_gone(lv, ["9154330e"]) == [] + assert rpc.deleted == [] diff --git a/simplyblock_core/test/test_repl_endgame_overhead.py b/simplyblock_core/test/test_repl_endgame_overhead.py index 7914e16ee9..afe973cf02 100644 --- a/simplyblock_core/test/test_repl_endgame_overhead.py +++ b/simplyblock_core/test/test_repl_endgame_overhead.py @@ -63,6 +63,10 @@ def get_clusters(self): def get_job_tasks(self, cluster_id, reverse=False): return list(self.tasks.values()) + def get_replication_tasks_for_snapshot(self, snapshot_id): + return [t for t in self.tasks.values() + if (t.function_params or {}).get("snapshot_id") == snapshot_id] + def get_task_by_id(self, uuid): if uuid not in self.tasks: raise KeyError(uuid) @@ -290,6 +294,17 @@ def test_a_failure_never_breaks_the_transfer(self): self.create.side_effect = RuntimeError("boom") self.assertIsNone(sr._prestage_next_landing(self.snap, self.remote_lv)) + def test_no_landing_volume_for_a_direct_round(self): + # bf29 (2026-11-20): a direct round is never replicated, so a landing + # volume staged under its name was never adopted nor discarded. + self._names("repl_round_next", "repl_round_next") + self.assertIsNone(sr._prestage_next_landing(self.snap, self.remote_lv)) + self.create.assert_not_called() + + def test_direct_round_prefix_matches_the_cutover_runner(self): + from simplyblock_core.services import tasks_runner_replication_final as trf + self.assertEqual(sr._DIRECT_ROUND_PREFIX, trf.DIRECT_ROUND_PREFIX) + class TestShrinkSnapshotNames(unittest.TestCase): diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 4b0df02ba8..66ef968e11 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -617,7 +617,8 @@ def test_interrupted_landing_volume_is_adopted_or_cleared(): src = inspect.getsource(sr.process_snap_replicate_start) probe = src.index('rep_name = f"REP_{snapshot.snap_name}"') create = src.index("_create_landing_volume(") - assert "lvol_controller.add_lvol_ha" in inspect.getsource(sr._create_landing_volume) + assert "lvol_controller.add_lvol_ha" in inspect.getsource(sr._create_landing_volume_unguarded) + assert "_create_landing_volume_unguarded(" in inspect.getsource(sr._create_landing_volume) assert probe < create, "the adopt/clear probe must run before the create" adopt = src.index("Adopting landing volume") assert probe < adopt < create diff --git a/simplyblock_core/test/test_replication_runner_resilience.py b/simplyblock_core/test/test_replication_runner_resilience.py index 87333f6db2..ebd4f4f7ad 100644 --- a/simplyblock_core/test/test_replication_runner_resilience.py +++ b/simplyblock_core/test/test_replication_runner_resilience.py @@ -144,6 +144,9 @@ def get_clusters(self): def get_job_tasks(self, cluster_id, reverse=False): return [_Task()] + def get_replication_tasks_for_snapshot(self, snapshot_id): + return [_Task()] + def get_task_by_id(self, uuid): raise KeyError(f"Task {uuid} not found") # vanished before the re-read diff --git a/simplyblock_core/test/test_round_landing_leak_bf29.py b/simplyblock_core/test/test_round_landing_leak_bf29.py new file mode 100644 index 0000000000..4d8094dff6 --- /dev/null +++ b/simplyblock_core/test/test_round_landing_leak_bf29.py @@ -0,0 +1,126 @@ +"""A round name that is replaced takes its pre-staged landing volume with it. + +Bed bf29 (2026-11-20): three 10 GiB REP_repl_round_* landing volumes stayed +on cluster A for good. The cutover names its next round snapshot in advance +(next_shrink_snap_name) and the replication runner pre-stages REP_. +_take_shrink_snapshot replaces that name when its kind does not match the +round (direct vs ordinary), when a snapshot of that name exists but is not +usable, and when the name turns out to be taken -- and left REP_ +behind each time. +""" +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.services import tasks_runner_replication_final as runner + +OLD = "repl_commit_old" +OLD_DIRECT = runner.DIRECT_ROUND_PREFIX + "old" + + +class _DB: + def __init__(self, landings, tasks=()): + self.landings = landings # name -> lvol + self.tasks = list(tasks) + + def get_lvol_by_name(self, name): + return self.landings[name] + + def get_job_tasks(self, cluster_id, **kw): + return list(self.tasks) + + +def _landing(lid, status=LVol.STATUS_ONLINE): + return SimpleNamespace(get_id=lambda: lid, status=status) + + +def _task(name): + t = JobSchedule() + t.uuid = "T1" + t.cluster_id = "A" + t.status = JobSchedule.STATUS_RUNNING + t.function_params = {"lvol_id": "LV1", "shrink_round": 2, "next_shrink_snap_name": name} + return t + + +def _lvol(): + lv = MagicMock() + lv.get_id.return_value = "LV1" + return lv + + +@pytest.fixture +def env(monkeypatch): + deleted = [] + from simplyblock_core.controllers import lvol_controller + monkeypatch.setattr(lvol_controller, "delete_lvol", + lambda lv, force_delete=False: deleted.append(lv.get_id())) + from simplyblock_core.controllers import replication_delta + add = MagicMock(return_value=("SNEW", None)) + monkeypatch.setattr(replication_delta, "add_snapshot", add) + monkeypatch.setattr(runner, "_volume_snapshots", lambda lv: []) + + def use(fake): + monkeypatch.setattr(runner, "db", fake) + return SimpleNamespace(deleted=deleted, add=add, use=use, monkeypatch=monkeypatch) + + +def test_bf29_a_direct_round_drops_the_landing_staged_for_an_ordinary_name(env): + env.use(_DB({f"REP_{OLD}": _landing("L-OLD")})) + task = _task(OLD) + _, err, _ = runner._take_shrink_snapshot(task.frozen_view(), _lvol(), direct=True) + assert err is None + assert env.deleted == ["L-OLD"] + assert env.add.call_args.args[1].startswith(runner.DIRECT_ROUND_PREFIX) + + +def test_bf29_an_ordinary_round_drops_the_landing_staged_for_a_direct_name(env): + env.use(_DB({f"REP_{OLD_DIRECT}": _landing("L-OLD")})) + runner._take_shrink_snapshot(_task(OLD_DIRECT).frozen_view(), _lvol(), direct=False) + assert env.deleted == ["L-OLD"] + + +def test_bf29_a_taken_name_drops_its_landing(env): + env.use(_DB({f"REP_{OLD}": _landing("L-OLD")})) + env.add.return_value = (None, "Snapshot name must be unique") + task = _task(OLD) + _, err, _ = runner._take_shrink_snapshot(task.frozen_view(), _lvol()) + assert "must be unique" in err + assert env.deleted == ["L-OLD"] + assert task.function_params["next_shrink_snap_name"] != OLD + + +def test_bf29_an_unusable_snapshot_of_that_name_drops_its_landing(env): + env.use(_DB({f"REP_{OLD}": _landing("L-OLD")})) + stale = SimpleNamespace(snap_name=OLD, status=SnapShot.STATUS_IN_DELETION) + env.monkeypatch.setattr(runner, "_volume_snapshots", lambda lv: [stale]) + runner._take_shrink_snapshot(_task(OLD).frozen_view(), _lvol()) + assert env.deleted == ["L-OLD"] + + +def test_bf29_a_landing_a_transfer_lands_in_is_kept(env): + busy = JobSchedule() + busy.uuid = "R1" + busy.function_name = JobSchedule.FN_SNAPSHOT_REPLICATION + busy.status = JobSchedule.STATUS_RUNNING + busy.function_params = {"remote_lvol_id": "L-OLD"} + env.use(_DB({f"REP_{OLD}": _landing("L-OLD")}, tasks=[busy])) + runner._take_shrink_snapshot(_task(OLD).frozen_view(), _lvol(), direct=True) + assert env.deleted == [] + + +def test_bf29_a_landing_not_online_is_left_alone(env): + env.use(_DB({f"REP_{OLD}": _landing("L-OLD", LVol.STATUS_IN_DELETION)})) + runner._take_shrink_snapshot(_task(OLD).frozen_view(), _lvol(), direct=True) + assert env.deleted == [] + + +def test_bf29_a_matching_name_keeps_its_landing_for_adoption(env): + env.use(_DB({f"REP_{OLD}": _landing("L-OLD")})) + runner._take_shrink_snapshot(_task(OLD).frozen_view(), _lvol(), direct=False) + assert env.deleted == [] + assert env.add.call_args.args[1] == OLD diff --git a/tests/unit/test_healthy_leaderless_recovery.py b/tests/unit/test_healthy_leaderless_recovery.py new file mode 100644 index 0000000000..d5e2d14c5a --- /dev/null +++ b/tests/unit/test_healthy_leaderless_recovery.py @@ -0,0 +1,273 @@ +"""A leaderless lvstore whose members are ALL online gets its leader back. + +bf32 2026-11-21, round R6 "tgt-dual-p shutdown" (target cluster 70edb423): +the primary and secondary of LVS_7 were shut down and restarted. While the +secondary's restart ran, the SPDK of two other nodes (8de868b9, 2aa1550c) went +away; at 05:16:27.95 a write to LVS_7 on its primary 484ff828 and one to +LVS_10 on its primary 9154330e failed, and each SPDK dropped the leadership +itself ("Queued failed IO and Leadership state changed internally to false +for lvs primary"). SPDK takes it back only on the next IO it receives, and +both lvstores were replication targets: replication waits for a leader before +it sends. The restarts of 8de868b9 and 2aa1550c then rebuilt LVS_10 as +non-leaders "behind" 9154330e without asking whether it led, nobody touched +LVS_7 again, and both stayed leaderless with every member online until a +manual _recover_leaderless_lvs at 06:34 (76 min). + +Now the node monitor (for the primary's own lvstore, after a confirmation +window) and a replication transfer waiting on such a target ask +storage_node_ops.restore_healthy_lvs_leadership, which grants only through the +guarded _recover_leaderless_lvs and only while every member is online and +answers non-leader. +""" +import types +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants, storage_node_ops +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_replication as sr +from simplyblock_core.services import storage_node_monitor as mon + + +def _node(nid, lvstore="", sec="", tert="", leader=False, status=StorageNode.STATUS_ONLINE): + n = MagicMock() + n.get_id.return_value = nid + n.status, n.lvstore, n.cluster_id = status, lvstore, "CL" + n.secondary_node_id, n.tertiary_node_id = sec, tert + n.is_secondary_node, n.jm_vuid = False, 7 if lvstore else 0 + n.health_check, n.lvstore_status, n.site = True, "ready", "" + n.leader = leader + n.rpc_client.return_value.get_lvstore.side_effect = ( + lambda name, _n=n: {"name": name, "lvs leadership": _n.leader}) + return n + + +class _Base(unittest.TestCase): + + def setUp(self): + # LVS_7: primary P (484ff828), secondary S (d10f59c9), tertiary T (9154330e) + self.p = _node("P", "LVS_7", "S", "T") + self.s = _node("S", "LVS_4", "T", "X") + self.t = _node("T", "LVS_10", "X", "Y") + self.nodes = {n.get_id(): n for n in (self.p, self.s, self.t)} + self.cluster = types.SimpleNamespace(status=Cluster.STATUS_ACTIVE) + db = MagicMock() + db.get_storage_node_by_id.side_effect = lambda i: self.nodes[i] + db.get_storage_nodes_by_cluster_id.return_value = list(self.nodes.values()) + db.get_cluster_by_id.return_value = self.cluster + self.recover = MagicMock(side_effect=lambda cid, members, lvs, taker: taker) + self.ops = patch.multiple( + storage_node_ops, DBController=MagicMock(return_value=db), + _cluster_two_node_arbitrated=MagicMock(return_value=False), + _recover_leaderless_lvs=self.recover) + self.ops.start() + self.addCleanup(self.ops.stop) + + +class TestHealthyLeadershipState(_Base): + + def test_leaderless_with_every_member_online(self): + state, members, why = storage_node_ops.healthy_lvs_leadership_state("P") + self.assertEqual(state, "leaderless") + self.assertEqual([m.get_id() for m in members], ["P", "S", "T"]) + self.assertIn("LVS_7", why) + + def test_led(self): + self.t.leader = True + state, leaders, _ = storage_node_ops.healthy_lvs_leadership_state("P") + self.assertEqual((state, [n.get_id() for n in leaders]), ("led", ["T"])) + + def test_a_member_not_online_is_not_ours(self): + for st in (StorageNode.STATUS_OFFLINE, StorageNode.STATUS_RESTARTING, + StorageNode.STATUS_DOWN): + self.s.status = st + state, _, why = storage_node_ops.healthy_lvs_leadership_state("P") + self.assertEqual(state, "skip", st) + self.assertIn("not online", why) + + def test_a_member_that_cannot_be_asked_may_lead(self): + self.s.rpc_client.return_value.get_lvstore.side_effect = RuntimeError("timeout") + state, _, why = storage_node_ops.healthy_lvs_leadership_state("P") + self.assertEqual(state, "skip") + self.assertIn("could not be asked", why) + + def test_not_while_the_cluster_is_suspended(self): + self.cluster.status = Cluster.STATUS_SUSPENDED + self.assertEqual(storage_node_ops.healthy_lvs_leadership_state("P")[0], "skip") + + def test_not_in_a_restart_window_or_with_a_failed_health_check(self): + self.p.lvstore_status = "in_creation" + self.assertEqual(storage_node_ops.healthy_lvs_leadership_state("P")[0], "skip") + self.p.lvstore_status, self.p.health_check = "ready", False + self.assertEqual(storage_node_ops.healthy_lvs_leadership_state("P")[0], "skip") + + def test_not_on_sync_or_two_node_clusters(self): + self.p.site = "site-a" + self.assertEqual(storage_node_ops.healthy_lvs_leadership_state("P")[0], "skip") + self.p.site = "" + with patch.object(storage_node_ops, "_cluster_two_node_arbitrated", return_value=True): + self.assertEqual(storage_node_ops.healthy_lvs_leadership_state("P")[0], "skip") + + def test_the_primary_is_found_by_lvstore_name(self): + # a volume's recorded node_id may not be its lvstore's primary + state, members, _ = storage_node_ops.healthy_lvs_leadership_state("S", "LVS_7") + self.assertEqual(state, "leaderless") + self.assertEqual(members[0].get_id(), "P") + + +class TestRestoreHealthyLeadership(_Base): + + def test_guarded_recovery_on_the_primary(self): + leader, why = storage_node_ops.restore_healthy_lvs_leadership("P", reason="test") + self.assertIs(leader, self.p) + args = self.recover.call_args.args + self.assertEqual((args[0], [m.get_id() for m in args[1]], args[2], args[3].get_id()), + ("CL", ["P", "S", "T"], "LVS_7", "P")) + self.assertIn("restored", why) + + def test_nothing_granted_when_led(self): + self.p.leader = True + leader, _ = storage_node_ops.restore_healthy_lvs_leadership("P") + self.assertIs(leader, self.p) + self.recover.assert_not_called() + + def test_nothing_granted_with_a_member_down(self): + self.t.status = StorageNode.STATUS_OFFLINE + leader, why = storage_node_ops.restore_healthy_lvs_leadership("P") + self.assertIsNone(leader) + self.recover.assert_not_called() + + def test_a_declined_recovery_reports_none(self): + self.recover.side_effect = None + self.recover.return_value = None + leader, why = storage_node_ops.restore_healthy_lvs_leadership("P") + self.assertIsNone(leader) + self.assertIn("granted nothing", why) + + +class TestMonitorRestoresLeadership(unittest.TestCase): + + def setUp(self): + for d in (mon._healthy_leaderless_checked, mon._healthy_leaderless_since): + d.clear() + self.addCleanup(d.clear) + mon._healthy_leaderless_running.clear() + self.p = _node("P", "LVS_7", "S", "T") + self.state = MagicMock(return_value=("leaderless", [self.p], "LVS_7: no member reports")) + self.restore = MagicMock(return_value=(self.p, "LVS_7: leadership restored on P")) + pt = patch.multiple(storage_node_ops, healthy_lvs_leadership_state=self.state, + restore_healthy_lvs_leadership=self.restore) + pt.start() + self.addCleanup(pt.stop) + # run the recovery thread inline + pt = patch.object(mon.threading, "Thread", side_effect=lambda target, **kw: types.SimpleNamespace(start=target)) + pt.start() + self.addCleanup(pt.stop) + self.clock = [1000.0] + pt = patch.object(mon.time, "monotonic", side_effect=lambda: self.clock[0]) + pt.start() + self.addCleanup(pt.stop) + + def _tick(self, dt): + self.clock[0] += dt + mon._restore_leadership_if_leaderless(self.p) + + def test_acts_only_after_the_confirmation_window(self): + self._tick(0) + self.restore.assert_not_called() + self._tick(mon.HEALTHY_LEADERLESS_CHECK_INTERVAL_SEC) + self.restore.assert_not_called() + self._tick(mon.HEALTHY_LEADERLESS_CONFIRM_SEC) + self.restore.assert_called_once() + self.assertEqual(self.restore.call_args.args, ("P",)) + self.assertNotIn(("CL", "LVS_7"), mon._healthy_leaderless_since) + + def test_probe_rate_limited(self): + self._tick(0) + self._tick(1) + self.assertEqual(self.state.call_count, 1) + + def test_any_other_observation_restarts_the_window(self): + self._tick(0) + self.state.return_value = ("led", [self.p], "led by P") + self._tick(mon.HEALTHY_LEADERLESS_CHECK_INTERVAL_SEC) + self.state.return_value = ("leaderless", [self.p], "") + self._tick(mon.HEALTHY_LEADERLESS_CONFIRM_SEC) + self.restore.assert_not_called() + + def test_only_for_a_primary(self): + self.p.is_secondary_node = True + self._tick(0) + self.p.is_secondary_node, self.p.lvstore = False, "" + self._tick(100) + self.state.assert_not_called() + + def test_a_failing_recovery_is_contained(self): + self.restore.side_effect = RuntimeError("boom") + self._tick(0) + self._tick(mon.HEALTHY_LEADERLESS_CONFIRM_SEC) + self.assertNotIn(("CL", "LVS_7"), mon._healthy_leaderless_running) + + +class TestReplicationAsksForRecovery(unittest.TestCase): + + def setUp(self): + self.task = types.SimpleNamespace( + uuid="66c78316", retry=0, status="running", function_result="", + function_params={"snapshot_id": "snap-1"}, write_to_db=MagicMock()) + self.remote_lv = types.SimpleNamespace( + node_id="484ff828", nodes=["484ff828", "d10f59c9", "9154330e"], + lvs_name="LVS_7", cluster_id="CL") + db = MagicMock() + db.get_storage_node_by_id.side_effect = ( + lambda i: types.SimpleNamespace(status="online")) + db.get_snapshot_by_id.return_value = types.SimpleNamespace(lvol="src-lvol") + for target, attr, new in ((sr, "db", db), + (sr.lvol_events, "replication_target_leaderless", MagicMock())): + pt = patch.object(target, attr, new) + pt.start() + self.addCleanup(pt.stop) + self.restore = MagicMock(return_value=(None, "declined")) + pt = patch.object(storage_node_ops, "restore_healthy_lvs_leadership", self.restore) + pt.start() + self.addCleanup(pt.stop) + + def _wait(self, age=None): + if age is not None: + self.task.function_params["target_leaderless_since"] -= age + sr._wait_for_target_leader(self.task, self.remote_lv, + "0 members of LVS_7 report leadership") + + def test_not_before_the_recover_delay(self): + self._wait() + self.restore.assert_not_called() + + def test_asks_once_the_wait_is_long_and_rate_limits(self): + self._wait() + self._wait(constants.REPL_TARGET_LEADERLESS_RECOVER_SEC + 1) + self.restore.assert_called_once() + self.assertEqual(self.restore.call_args.args, ("484ff828", "LVS_7")) + self._wait() + self.restore.assert_called_once() # rate-limited + self.task.function_params["target_recover_tried_at"] -= ( + constants.REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC + 1) + self._wait() + self.assertEqual(self.restore.call_count, 2) + self.assertEqual(self.task.status, "suspended") # still waits + + def test_a_failing_recovery_does_not_break_the_wait(self): + self.restore.side_effect = RuntimeError("fdb") + self._wait() + self._wait(constants.REPL_TARGET_LEADERLESS_RECOVER_SEC + 1) + self.assertEqual(self.task.status, "suspended") + + def test_a_leader_clears_the_rate_limit(self): + self._wait() + self._wait(constants.REPL_TARGET_LEADERLESS_RECOVER_SEC + 1) + sr._target_leader_found(self.task) + self.assertNotIn("target_recover_tried_at", self.task.function_params) + + +if __name__ == "__main__": + unittest.main()