From daea21f5f7068b3f3baea134223fa7fd8b56ba46 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 19 Nov 2026 13:04:26 +0100 Subject: [PATCH 1/9] fix(replication): one client under two host NQNs is not a stranded host #1674's guard compared host NQNs between the side whose namespace goes and the side that stays. A Kubernetes node connects with the simplyblock host NQN (node UID) on one path and with the kernel default on another: on bed bf28 every client reached the source as nqn.2014-08.io.simplyblock:uuid: and the early-prepared target as nqn.2014-08.org.nvmexpress:uuid:. The guard took each as a different host, so every cutover freeze waited for a reconnect that had already happened, and R1 stopped at its first three cutovers; a discarded target clone was held the same way. Nothing records which host NQNs belong to one machine, so hosts are matched by name where possible and every unmatched host on the keeping side counts for one unmatched host on the removing side. A host is stranded only when the removing side has more unmatched hosts than that. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/controllers/ns_host_guard.py | 23 ++++++++++++++++--- simplyblock_core/test/test_ns_host_guard.py | 19 +++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/simplyblock_core/controllers/ns_host_guard.py b/simplyblock_core/controllers/ns_host_guard.py index 78f71db259..e7a0981585 100644 --- a/simplyblock_core/controllers/ns_host_guard.py +++ b/simplyblock_core/controllers/ns_host_guard.py @@ -62,8 +62,22 @@ def connected_hosts(nqn: str, node_ids: Iterable[str], db=None) -> set | None: def stranded_hosts(nqn: str, removing: Iterable[str], keeping: Iterable[str], db=None) -> set | None: - """Hosts connected to *nqn* on the *removing* nodes but on none of the - *keeping* nodes; None when that could not be established.""" + """Hosts connected to *nqn* on the *removing* nodes that have no controller + on the *keeping* nodes; None when that could not be established. + + A host NQN does not identify a machine: one Kubernetes node connects with + the kernel default (``/etc/nvme/hostnqn``) on one path and with the + simplyblock form derived from its node UID on another (bed bf28, + 2026-11-19: every client reached the source as + ``nqn.2014-08.io.simplyblock:uuid:`` and the early-prepared + target as ``nqn.2014-08.org.nvmexpress:uuid:``, with different + host ids), and nothing records which forms belong together. So hosts are + matched by name where they can be, and every unmatched host on the keeping + side is taken to be one unmatched host of the removing side under its + other name. Hosts are stranded only when the removing side has more + unmatched hosts than the keeping side can account for; all of the removing + side's unmatched hosts are then reported, since which of them is the + stranded one cannot be told.""" gone = connected_hosts(nqn, removing, db=db) if gone is None: return None @@ -72,7 +86,10 @@ def stranded_hosts(nqn: str, removing: Iterable[str], keeping: Iterable[str], stay = connected_hosts(nqn, keeping, db=db) if stay is None: return None - return gone - stay + unmatched_gone, unmatched_stay = gone - stay, stay - gone + if len(unmatched_gone) <= len(unmatched_stay): + return set() + return unmatched_gone def removal_blocked(nqn: str, removing: Iterable[str], keeping: Iterable[str], diff --git a/simplyblock_core/test/test_ns_host_guard.py b/simplyblock_core/test/test_ns_host_guard.py index 3da5df761d..bd97e8bf26 100644 --- a/simplyblock_core/test/test_ns_host_guard.py +++ b/simplyblock_core/test/test_ns_host_guard.py @@ -59,6 +59,25 @@ def test_bed_bf26_a_host_only_on_the_side_that_goes_is_stranded(guard): assert H1 in why and "discarding clone C" in why +def test_bed_bf28_one_machine_under_two_host_nqns_is_not_stranded(guard): + # Source paths carry the simplyblock host NQN, the early-prepared target + # paths the kernel default: the same client, never stranded. + sb = "nqn.2014-08.io.simplyblock:uuid:5892b49a-bc39-41b2-a38a-8daae13a9d0d" + kernel = "nqn.2014-08.org.nvmexpress:uuid:ec2d55d6-1c18-ced7-92ee-34f409cb58ce" + db = _DB(_node("S1", [sb]), _node("S2", [sb]), _node("T1", [kernel]), _node("T2", [kernel])) + # the cutover freeze: the source paths go inaccessible, the target ones stay + assert guard.stranded_hosts(NQN, ["S1", "S2"], ["T1", "T2"], db=db) == set() + # a discarded target clone: the target paths go, the source ones stay + assert guard.removal_blocked(NQN, ["T1", "T2"], ["S1", "S2"], "x", db=db) is None + + +def test_more_unmatched_hosts_going_than_staying_are_stranded(guard): + # Two clients on the side that goes, one other name on the side that stays: + # at least one machine loses its last path; both candidates are named. + db = _DB(_node("T1", [H1, H2]), _node("S1", ["nqn.host:other"])) + assert guard.stranded_hosts(NQN, ["T1"], ["S1"], db=db) == {H1, H2} + + def test_nodes_that_are_down_or_lack_the_subsystem_serve_no_host(guard): db = _DB(_node("T1", [H1]), _node("S1", [H1], status=StorageNode.STATUS_OFFLINE), From 6eac64107d047074fb8ded19ccfc581937ebb0b1 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 03:13:23 +0100 Subject: [PATCH 2/9] fix(replication): the source-delete guard finds the target by NQN After a cutover the prepared clone named in tgt_lvol_id no longer exists: the volume runs on the other cluster under a new record with the same NQN. Bed bf29 (2026-11-20, round 2): the guard looked the target up by tgt_lvol_id, found nothing, took every host as stranded and after 120 s kept the migrated-away source of two volumes. That leaks a full volume per migration and leaves a same-named volume on the cluster the next round migrates back to. _live_target_node_ids: every other live volume exporting the source's NQN, plus the clone while it still exists. Co-Authored-By: Claude Opus 5.5 --- .../tasks_runner_replication_final.py | 24 ++++++++++++++++++- simplyblock_core/test/test_ns_host_guard.py | 22 +++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index f03dbe23ea..836a90ff02 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -360,7 +360,7 @@ def _delete_source_if_requested(task): while True: blocked = ns_host_guard.removal_blocked( getattr(src_lvol, "nqn", ""), ns_host_guard.volume_node_ids(src_lvol), - _target_node_ids(task.function_params), + _live_target_node_ids(src_lvol, task.function_params), f"deleting migrated-away source {src_lvol_id}") if not blocked or time.time() >= deadline: break @@ -2121,6 +2121,28 @@ def _target_node_ids(params): return [] +def _live_target_node_ids(src_lvol, params): + """The nodes exporting the volume the source migrated to. + + After the cutover the prepared clone's record is gone: the volume runs + under a new record carrying the same NQN. Bed bf29 (2026-11-20): the + source-delete guard looked the target up by tgt_lvol_id, found nothing, + took every host as stranded and kept the migrated-away source of two + volumes (a full volume leaked per migration, and a same-named volume left + on the cluster the next round migrates back to). So the target is found + by NQN: every other live volume exporting it, plus the clone if it still + exists.""" + ids = list(_target_node_ids(params)) + nqn = getattr(src_lvol, "nqn", "") + if nqn: + for cluster in db.get_clusters(): + for lv in db.get_lvols(cluster.get_id()): + if (lv.get_id() != src_lvol.get_id() and getattr(lv, "nqn", "") == nqn + and lv.status not in (LVol.STATUS_IN_DELETION, LVol.STATUS_DELETED)): + ids.extend(ns_host_guard.volume_node_ids(lv)) + return list(dict.fromkeys(ids)) + + def _source_node_ids(lvol_id): try: return ns_host_guard.volume_node_ids(db.get_lvol_by_id(lvol_id)) diff --git a/simplyblock_core/test/test_ns_host_guard.py b/simplyblock_core/test/test_ns_host_guard.py index bd97e8bf26..728894618f 100644 --- a/simplyblock_core/test/test_ns_host_guard.py +++ b/simplyblock_core/test/test_ns_host_guard.py @@ -146,3 +146,25 @@ def blocked(nqn, removing, keeping, what, db=None): src = inspect.getsource(runner) i = src.index("cutover freeze of") assert i < src.index("freeze_started_at=time.time()") + + +def test_bed_bf29_the_source_delete_finds_the_target_by_nqn_after_the_clone_is_gone(monkeypatch): + # After the cutover the clone named in tgt_lvol_id no longer exists; the + # volume runs on the other cluster under a new record with the same NQN. + src = SimpleNamespace(get_id=lambda: "SRC", nqn=NQN, node_id="S1", nodes=["S2"], + status="online") + live = SimpleNamespace(get_id=lambda: "NEW", nqn=NQN, node_id="T1", nodes=["T2", "T3"], + status="online") + other = SimpleNamespace(get_id=lambda: "X", nqn="nqn.other", node_id="T9", nodes=[], + status="online") + going = SimpleNamespace(get_id=lambda: "OLD", nqn=NQN, node_id="T7", nodes=[], + status="in_deletion") + vols = {"A": [src], "B": [live, other, going]} + + def get_lvol_by_id(i): + raise KeyError(i) + monkeypatch.setattr(runner, "db", SimpleNamespace( + get_lvol_by_id=get_lvol_by_id, + get_clusters=lambda: [SimpleNamespace(get_id=lambda: "A"), SimpleNamespace(get_id=lambda: "B")], + get_lvols=lambda cid: vols[cid])) + assert runner._live_target_node_ids(src, {"tgt_lvol_id": "GONE"}) == ["T1", "T2", "T3"] From 560d203e754ded78654829c46860b010af1c6a4d Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 14:15:17 +0100 Subject: [PATCH 3/9] fix(replication): a migration can no longer fill its source cluster Bed bf29 (2026-11-20): five volumes of cluster A ran their cutover endgame at once under sustained fio overwrite. Every shrink round snapshotted its volume every 15-30 s, every snapshot pinned what was overwritten after it, and the superseded snapshots were deleted more slowly than new ones were taken: A went 41% -> 82% -> 99% in 16 minutes and ran out of space. Three rules (simplyblock_core/migration_space.py) bound what a migration consumes on its source. They hold back only the migration's own snapshots, never client IO or client volume creation; cap_warn/cap_crit stay alarms. 1. At most REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER (1) endgames per source cluster (_acquire_lvs_claim). A queued volume holds no lvstore and keeps its ordinary cadence. The claim is re-checked against a fresh task read after it is written and ranked by claim time, so volumes that catch up in the same runner pass cannot all take the slot, and a running endgame is never displaced. 2. An endgame takes its next round snapshot only once none of the volume's snapshots is still in deletion (_round_snapshot_hold). A deletion stuck for REPL_ENDGAME_DELETION_WAIT_MAX_SEC (300 s) is set aside, logged. 3. No migration snapshot - endgame round, or cadence snapshot of a migration-mode volume (snapshot_monitor) - while the source's free space is under max(5% of the cluster, 2 x bytes written since the volume's last snapshot). Warning event once per wait. A missing or stale (>120 s) capacity record holds for at most 120 s, then the snapshot proceeds. The endgame does not take the lvstore claim while its first round would wait. A round measured on a pass that then could not snapshot is not re-measured with the waiting time on the next pass. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/constants.py | 29 ++ simplyblock_core/controllers/lvol_events.py | 23 + simplyblock_core/migration_space.py | 195 ++++++++ simplyblock_core/services/snapshot_monitor.py | 46 ++ .../tasks_runner_replication_final.py | 206 +++++++- simplyblock_core/test/conftest.py | 12 + .../test/test_migration_space_bf29.py | 452 ++++++++++++++++++ 7 files changed, 956 insertions(+), 7 deletions(-) create mode 100644 simplyblock_core/migration_space.py create mode 100644 simplyblock_core/test/test_migration_space_bf29.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 971c831f28..f9094cc9cf 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -711,6 +711,35 @@ def get_config_var(name, default=None): # released, the volume replicates on its ordinary cadence, and the endgame is # retried after this long (the abandon burns one attempt of the task's budget). REPL_CUTOVER_ENDGAME_BACKOFF_SEC = 300 +# --- a migration must not fill its source cluster (bed bf29, 2026-11-20) --- +# Five endgames of one source cluster at once, under sustained overwrite, took +# a shrink snapshot every 15-30 s each; every one pinned what the client +# overwrote after it, and their deletions fell behind: 41% -> 99% in 16 min. +# See simplyblock_core/migration_space.py. These hold back only the +# migration's own snapshots -- never client IO or client volume creation. +# +# Cutover endgames (shrink-round phase) running at once per SOURCE cluster. +# The others replicate on their ordinary cadence while they wait for a slot. +REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER = 1 +# An endgame takes its next round snapshot only once none of the volume's +# snapshots is still in deletion -- for at most this long per wait: a deletion +# that is stuck (no leader, a clone in deletion) must not stall the endgame +# into its shrink deadline on its own. The free-space check still applies. +REPL_ENDGAME_DELETION_WAIT_MAX_SEC = 300 +# A migration snapshot needs this many times the bytes written to the volume +# since its last snapshot free on the source (the snapshot pins them, and the +# client overwrites about as much again while the round transfers) ... +REPL_MIGRATION_HEADROOM_FACTOR = 2 +# ... and never less than this share of the cluster (other volumes' writes, +# and the capacity record being up to a collector interval old). +REPL_MIGRATION_MIN_FREE_PCT = 5 +# A cluster capacity record older than this is no reading (the collector +# writes one every DEV_STAT_COLLECTOR_INTERVAL_SEC). +REPL_MIGRATION_CAPACITY_STALE_SEC = 120 +# Without a usable capacity reading a migration snapshot waits this long, then +# goes ahead without the check (logged): a broken stats collector must not +# stop migrations for good. +REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC = 120 # --- cutover source throttle ---------------------------------------------- # A round converges to overhead / (1 - W/T): with the application writing at # W and the delta transferring at T, no number of rounds gets under the diff --git a/simplyblock_core/controllers/lvol_events.py b/simplyblock_core/controllers/lvol_events.py index 3b0459661f..e9f572ee7f 100644 --- a/simplyblock_core/controllers/lvol_events.py +++ b/simplyblock_core/controllers/lvol_events.py @@ -199,3 +199,26 @@ def replication_target_leaderless(lvol, lvs_name, waited_s, why, caused_by=ec.CA f"{lvs_name} has had no leader for {int(waited_s)}s ({why})"), node_id=lvol.get_id(), event_level=EventObj.LEVEL_ERROR) + + +def migration_space_low(lvol, reason, caused_by=ec.CAUSED_BY_MONITOR): + """A migration snapshot of *lvol* waits for free space on its source + cluster (simplyblock_core/migration_space.py). Warning level: client IO + is not affected, only the migration's own snapshots wait. Raised once per + wait, when it starts.""" + from simplyblock_core.models.events import EventObj + try: + cluster_id = DBController().get_pool_by_id(lvol.pool_uuid).cluster_id + except Exception as e: + logger.error(f"No cluster could be resolved for lvol {lvol.get_id()}; event not logged: {e}") + return + ec.log_event_cluster( + cluster_id=cluster_id, + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_REPLICATION_STALLED, + db_object=lvol, + caused_by=caused_by, + message=(f"Migration of LVol {lvol.get_id()} is waiting for free space on its " + f"source cluster: {reason}"), + node_id=lvol.get_id(), + event_level=EventObj.LEVEL_WARN) diff --git a/simplyblock_core/migration_space.py b/simplyblock_core/migration_space.py new file mode 100644 index 0000000000..2d91c5a93a --- /dev/null +++ b/simplyblock_core/migration_space.py @@ -0,0 +1,195 @@ +"""A migration must never fill its SOURCE cluster by itself. + +Bed bf29 (2026-11-20): five volumes of one source cluster ran their cutover +endgame at the same time, under sustained client overwrite. Every shrink +round takes a snapshot of the source volume, and every snapshot pins what the +client overwrites after it; the superseded ones were deleted (async on the +primary, sync on the secondaries) more slowly than new ones were taken. The +source went 41% -> 82% -> 99% in 16 minutes (~440 MB/s of pinned data) and +ran out of space. + +Three rules bound what a migration consumes on its source. Only the +migration's OWN snapshots are held back by them -- never a client write, and +never a client volume creation: the capacity thresholds (cap_warn/cap_crit) +stay alarms, and a cluster is full at 100 %. + +1. One endgame per source cluster at a time + (``REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER``; tasks_runner_replication_final. + _acquire_lvs_claim). The others keep replicating on their ordinary cadence + meanwhile -- an endgame queued for the slot holds no lvstore. +2. An endgame takes its next round snapshot only once the deletions of the + volume's earlier snapshots have completed (:func:`deletions_pending`). +3. No migration snapshot -- endgame round or ordinary cadence of a + migration-mode volume -- while the source's free space is under the + headroom estimate (:func:`space_hold`). +""" +import logging +import time + +from simplyblock_core import constants +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.storage_node import StorageNode + +logger = logging.getLogger(__name__) + + +def source_cluster_id(db, lvol, default=""): + """The cluster *lvol* lives on (its host node's), else *default*.""" + try: + return db.get_storage_node_by_id(lvol.node_id).cluster_id or default + except Exception: # noqa: BLE001 + return default + + +def cluster_free(db, cluster_id, now=None): + """``(free bytes, total bytes, "")`` from the cluster's latest capacity + record, or ``(None, None, why)`` when there is no usable one: none at + all, unreadable, no total, or older than REPL_MIGRATION_CAPACITY_STALE_SEC + (the collector writes one every DEV_STAT_COLLECTOR_INTERVAL_SEC; a record + much older than that says the collector is down, not that space is + free).""" + now = time.time() if now is None else now + try: + cluster = db.get_cluster_by_id(cluster_id) + records = db.get_cluster_capacity(cluster, 1) + except Exception as e: # noqa: BLE001 + return None, None, f"capacity of cluster {cluster_id} unreadable: {e}" + if not records: + return None, None, f"no capacity record for cluster {cluster_id}" + rec = records[0] + total = int(getattr(rec, "size_total", 0) or 0) + if total <= 0: + return None, None, f"capacity record of cluster {cluster_id} has no total" + age = now - float(getattr(rec, "date", 0) or 0) + if age > constants.REPL_MIGRATION_CAPACITY_STALE_SEC: + return None, None, (f"capacity record of cluster {cluster_id} is {int(age)}s " + f"old (limit {constants.REPL_MIGRATION_CAPACITY_STALE_SEC}s)") + used = int(getattr(rec, "size_used", 0) or 0) + return max(0, total - used), total, "" + + +def dirty_bytes(db, lvol): + """Bytes written to *lvol* since its newest snapshot, or None. + + The dirty bitmap of the volume's top bdev + (``bdev_lvol_dirty_bitmap_info``). Every online member is asked and the + largest complete count wins: only the leader's is the real one, and an + over-estimate only makes the hold more careful.""" + from simplyblock_core.controllers.replication_delta import _members + best = None + try: + members = _members(db, lvol) + except Exception: # noqa: BLE001 + return None + for node in members: + if getattr(node, "status", None) != StorageNode.STATUS_ONLINE: + continue + try: + info = node.rpc_client(timeout=constants.REPL_FINAL_DELTA_COUNTS_TIMEOUT_SEC, + retry=0).bdev_lvol_dirty_bitmap_info(lvol.top_bdev) + except Exception: # noqa: BLE001 + continue + if isinstance(info, dict) and info.get("tracking") and info.get("complete"): + n = int(info.get("dirty_bytes") or 0) + best = n if best is None else max(best, n) + return best + + +def headroom_floor(total): + """The free space a migration snapshot always leaves: REPL_MIGRATION_MIN_FREE_PCT + of the cluster.""" + return int(total * constants.REPL_MIGRATION_MIN_FREE_PCT / 100) + + +def required_free(total, dirty): + """Free space a migration snapshot needs: REPL_MIGRATION_HEADROOM_FACTOR + times what the snapshot pins at once (the bytes written since the + volume's last snapshot), and never less than :func:`headroom_floor`. + + Why x2: the new snapshot pins those bytes the moment it is taken, and + while its round transfers the client overwrites about as much again -- + new allocations that the previous snapshot, still pending deletion, keeps + from being reused. The floor covers what one volume's count cannot see: + the other volumes' writes in between, and that the capacity record is up + to one collector interval old (at bf29's 440 MB/s, 15 s are 6.6 GB).""" + return max(headroom_floor(total), + int(constants.REPL_MIGRATION_HEADROOM_FACTOR * max(0, dirty or 0))) + + +def assess(db, lvol, cluster_id, now=None): + """``("ok" | "low" | "unknown", detail)`` for a migration snapshot of + *lvol* on *cluster_id*. Asks SPDK for the volume's dirty count only when + the answer depends on it.""" + free, total, why = cluster_free(db, cluster_id, now) + if free is None: + return "unknown", why + floor = headroom_floor(total) + if free < floor: + return "low", (f"{free >> 30} GiB free on the source cluster, under the " + f"{constants.REPL_MIGRATION_MIN_FREE_PCT}% floor ({floor >> 30} GiB)") + # The dirty count cannot exceed the volume: with that much room there is + # nothing to ask. + size = int(getattr(lvol, "size", 0) or 0) + if size and free >= max(floor, constants.REPL_MIGRATION_HEADROOM_FACTOR * size): + return "ok", "" + dirty = dirty_bytes(db, lvol) + if dirty is None: + # Unknown dirty count: the worst case is the whole volume. + dirty = size + need = required_free(total, dirty) + if free < need: + return "low", (f"{free >> 30} GiB free on the source cluster, under the " + f"{need >> 30} GiB headroom ({constants.REPL_MIGRATION_HEADROOM_FACTOR}x " + f"{dirty >> 20} MiB written since the last snapshot, floor " + f"{floor >> 30} GiB)") + return "ok", "" + + +def space_hold(db, lvol, cluster_id, unknown_since=0, now=None): + """Whether a migration snapshot of *lvol* must wait for space. + + Returns ``(reason, unknown_since)``: an empty reason means take it. A + capacity that cannot be read holds the snapshot for at most + REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC from *unknown_since* (the caller + keeps it and hands it back) and then lets it through, loudly: a broken + stats collector must not stop every migration for good. Space that is + known to be short holds for as long as it lasts -- the endgame's own + shrink deadline bounds that wait, and a cadence snapshot just waits.""" + now = time.time() if now is None else now + verdict, detail = assess(db, lvol, cluster_id, now) + if verdict == "ok": + return "", 0 + if verdict == "low": + return f"migration snapshot of {lvol.get_id()} deferred: {detail}", 0 + since = unknown_since or now + waited = now - since + if waited >= constants.REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC: + logger.warning("migration snapshot of %s: source capacity unknown for %ds " + "(%s); proceeding without the free-space check", + lvol.get_id(), int(waited), detail) + return "", since + return (f"migration snapshot of {lvol.get_id()} deferred: {detail}; " + f"proceeding without the check in " + f"{int(constants.REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC - waited)}s"), since + + +def deletions_pending(db, lvol): + """Ids of *lvol*'s snapshots whose deletion has not completed yet + (status in_deletion). A soft-deleted snapshot (``deleted`` but online, + held by a clone) is not pending: nothing is deleting it.""" + try: + snaps = db.get_snapshots_by_lvol_id(lvol.get_id()) + except Exception as e: # noqa: BLE001 + logger.warning("Cannot read the snapshots of %s: %s", lvol.get_id(), e) + return [] + return [s.get_id() for s in snaps if s.status == SnapShot.STATUS_IN_DELETION] + + +def space_low_event(lvol, reason): + """A warning-level cluster event: a migration snapshot waits for space.""" + try: + from simplyblock_core.controllers import lvol_events + lvol_events.migration_space_low(lvol, reason) + except Exception as e: # noqa: BLE001 + logger.warning("Could not log the migration space event for %s: %s", + lvol.get_id(), e) diff --git a/simplyblock_core/services/snapshot_monitor.py b/simplyblock_core/services/snapshot_monitor.py index 0251602c65..96beba07d5 100644 --- a/simplyblock_core/services/snapshot_monitor.py +++ b/simplyblock_core/services/snapshot_monitor.py @@ -6,6 +6,7 @@ from simplyblock_core import ( constants, db_controller, + migration_space, migration_turn, replication_spare, utils, @@ -910,6 +911,39 @@ def _cutover_endgame_lvol_ids(cluster_id): return ids +# Per volume: since when its source capacity could not be read, and the +# volumes whose cadence currently waits for space (one event per wait). +_space_unknown_since: dict = {} +_space_low: set = set() + + +def _migration_space_hold(lvol, cluster_id, now_ts): + """Why the cadence snapshot of a migration-mode volume must wait for free + space on its source cluster, or "" (rule 3 of + simplyblock_core/migration_space.py, bed bf29 2026-11-20). Only a + migration's own snapshots wait: a volume replicating for DR is not + affected, and neither is any client write.""" + if getattr(lvol, "replication_mode", "") != "migration": + return "" + vid = lvol.get_id() + why, since = migration_space.space_hold( + db, lvol, cluster_id, _space_unknown_since.get(vid, 0), now_ts) + if since: + _space_unknown_since[vid] = since + else: + _space_unknown_since.pop(vid, None) + if why: + if vid not in _space_low: + _space_low.add(vid) + migration_space.space_low_event(lvol, why) + return why + if vid in _space_low: + _space_low.discard(vid) + logger.info("Cadence snapshots of migrating lvol %s resume: source space " + "recovered", vid) + return "" + + def take_due_internal_snapshots(cluster_id, now_ts): """Create an internal snapshot for every replicated volume whose interval has elapsed. The snapshot's creation auto-enqueues a replication task. @@ -1022,6 +1056,14 @@ def _group_id_of(lvol): break if held: continue + for lv in members: + held = _migration_space_hold(lv, cluster_id, now_ts) + if held: + logger.warning("Skipping group snapshot for group %s: %s", + group_id, held) + break + if held: + continue group = db.get_consistency_group_by_id(group_id) # A group is pinned to one LVS: one lookup defers it as a whole. if sync and _cadence_waits_for_leader(group.node_id, group.lvs_name, leaderless): @@ -1063,6 +1105,10 @@ def _group_id_of(lvol): logger.warning("Skipping internal replication snapshot for lvol %s: " "replication held: %s", lvol.get_id(), held) continue + held = _migration_space_hold(lvol, cluster_id, now_ts) + if held: + logger.warning("Skipping internal replication snapshot: %s", held) + continue name = f"repl_internal_{lvol.get_id()[:8]}_{now_ts}" logger.info(f"Taking internal replication snapshot for lvol {lvol.get_id()}: {name}") _id, err = replication_delta.add_snapshot( diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index 836a90ff02..f37487fb29 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -25,7 +25,13 @@ from datetime import datetime from typing import Any, NoReturn -from simplyblock_core import constants, db_controller, utils, xfer_timing +from simplyblock_core import ( + constants, + db_controller, + migration_space, + utils, + xfer_timing, +) from simplyblock_core.controllers import ns_host_guard from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol, LVolReplication @@ -137,7 +143,8 @@ def _release_lvs_claim(task): before this runs. """ _requeue_direct_rounds(task) - drop_params(task, "cutover_lvs", "cutover_group", "throttle_tried") + drop_params(task, "cutover_lvs", "cutover_group", "cutover_cluster", + "cutover_claimed_at", "throttle_tried") def _throttle_cap(task): @@ -578,6 +585,10 @@ def _acquire_lvs_claim(task, lvol, tasks): int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC)) raise TaskDefer(f"queued for lvstore {lvs_name} behind " f"{owner_lvol[:8]} (delta already converged)") + cluster_id = migration_space.source_cluster_id(db, lvol, task.cluster_id) + busy = _cluster_endgame_slots_taken(task, tasks, own_group, cluster_id) + if busy: + raise TaskDefer(busy) xfer_timing.stamp("lvs_claim_acquired", lvol=lvol.get_id(), lvs=lvs_name, round=params.get("shrink_round")) # Committed immediately: the claim has to be visible to the other hosts' @@ -585,7 +596,93 @@ def _acquire_lvs_claim(task, lvol, tasks): # endgame_started_at survives a released-and-retaken claim: the plan # forecast measures a cutover from its FIRST claim to done. extra = {} if params.get("endgame_started_at") else {"endgame_started_at": time.time()} - return _record(task, cutover_lvs=lvs_name, cutover_group=own_group, **extra) + task = _record(task, cutover_lvs=lvs_name, cutover_group=own_group, + cutover_cluster=cluster_id, cutover_claimed_at=time.time(), **extra) + return _confirm_cluster_endgame_slot(task, own_group, cluster_id) + + +def _endgame_unit(t): + """What one endgame slot holds: a consistency group cuts over as one.""" + params = t.function_params or {} + return params.get("cutover_group") or t.get_id() + + +def _cluster_endgame_holders(tasks, cluster_id): + """The FN_REPLICATION_FINAL tasks of *tasks* that hold an endgame of + *cluster_id* (a claimed lvstore), live ones only.""" + out = [] + for t in tasks or (): + if t.function_name != JobSchedule.FN_REPLICATION_FINAL: + continue + if t.status == JobSchedule.STATUS_DONE or t.canceled: + continue + params = t.function_params or {} + if not params.get("cutover_lvs"): + continue + # a claim from before the cluster was recorded counts where it runs + if (params.get("cutover_cluster") or t.cluster_id) != cluster_id: + continue + out.append(t) + return out + + +def _cluster_endgame_slots_taken(task, tasks, own_group, cluster_id): + """Why this volume may not start an endgame on its source cluster now, + or "" (rule 1 of simplyblock_core/migration_space.py). + + At most REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER endgames run at once per + source cluster, whatever lvstores they are on: each one takes a snapshot + of its volume every round, and bf29 (2026-11-20) ran five at once and + filled the source. A volume that waits here has taken no snapshot and + holds no lvstore, so it keeps replicating on its ordinary cadence. A + consistency-group sibling joins its group's slot.""" + limit = max(1, int(constants.REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER)) + units = {_endgame_unit(t) for t in _cluster_endgame_holders(tasks, cluster_id) + if t.get_id() != task.get_id()} + if own_group and own_group in units: + return "" + if len(units) < limit: + return "" + _record(task, shrink_deadline=( + int(time.time()) + constants.REPL_CUTOVER_SHRINK_TIMEOUT_SEC)) + return (f"queued for an endgame slot of source cluster {cluster_id}: " + f"{len(units)} of {limit} in use") + + +def _confirm_cluster_endgame_slot(task, own_group, cluster_id): + """Re-check the cluster's endgame slots after the claim is committed. + + The pre-check reads the pass's task list, which predates every claim made + in this pass: volumes that all caught up in the same pass would each see + the slots free. Every claimant writes its claim BEFORE this read, so of + two racing claimants at least one sees the other, and both rank the + claims the same way -- by claim time, so a running endgame is never + displaced by a later claim. A claimant ranked past the limit withdraws. + An unreadable task table leaves the pre-check's answer standing.""" + limit = max(1, int(constants.REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER)) + try: + fresh = db.get_job_tasks(task.cluster_id) + except Exception as e: # noqa: BLE001 + logger.warning("cutover for lvol %s: cannot re-check the endgame slots of " + "cluster %s: %s", task.function_params.get("lvol_id"), cluster_id, e) + return task + first: dict[str, tuple[float, str]] = {} + for t in _cluster_endgame_holders(fresh, cluster_id) + [task]: + params = t.function_params or {} + key = (float(params.get("cutover_claimed_at") or 0), t.get_id()) + unit = _endgame_unit(t) + if unit not in first or key < first[unit]: + first[unit] = key + ranked = sorted(first, key=lambda u: first[u]) + own_unit = own_group or task.get_id() + if own_unit in ranked[:limit]: + return task + logger.info("cutover for lvol %s: endgame slot of cluster %s lost to a concurrent " + "claim; withdrawing", task.function_params.get("lvol_id"), cluster_id) + drop_params(task, "cutover_lvs", "cutover_group", "cutover_cluster", + "cutover_claimed_at") + raise TaskDefer(f"queued for an endgame slot of source cluster {cluster_id}: " + f"lost to a concurrent claim") def _await_cutover_proceed(task, params): @@ -756,6 +853,13 @@ def task_runner(task: JobSchedule, tasks): task = drop_params(task, "catchup_since", "catchup_win", "catchup_prev", "catchup_strikes") or task params = task.function_params + # Nothing is claimed while the first round could not be taken anyway + # (pending deletions, a source short of space): holding the lvstore + # and the cluster's endgame slot through that wait would stop the + # other volumes' transfers for nothing. The round checks again. + held, task = _round_snapshot_hold(task, lvol) + if held: + raise TaskDefer(held) task = _acquire_lvs_claim(task, lvol, tasks) # The endgame gets its full convergence budget from here. The deadline # an abandon set (back-off + timeout) also had to cover the prune gate @@ -1337,11 +1441,88 @@ def _direct_last_round(params): return None +def _round_snapshot_hold(task, lvol): + """``(why, task)``: why the next round snapshot must wait, or "". + + Rules 2 and 3 of simplyblock_core/migration_space.py (bed bf29, + 2026-11-20: five endgames' round snapshots pinned ~440 MB/s of + overwritten data and filled the source cluster): + + * Rule 2 -- no new round snapshot while one of the volume's snapshots is + still being deleted: rounds then go only as fast as their predecessors + are released, instead of stacking pending deletions. A deletion still + pending after REPL_ENDGAME_DELETION_WAIT_MAX_SEC is set aside (logged) + and no longer waited for; any newer one is. + * Rule 3 -- no new round snapshot while the source cluster's free space + is under the headroom the snapshot needs (migration_space.space_hold). + + Only this migration's snapshots wait; client IO is never held. The wait + is bounded by the endgame's shrink deadline like any other: an endgame + that cannot snapshot is abandoned, releasing its slot and lvstore. + """ + params = task.function_params + now = time.time() + lvol_id = lvol.get_id() + nxt = params.get("shrink_round", 0) + 1 + + # ---- rule 2: the volume's earlier deletions have completed ----------- # + ignored = set(params.get("deletion_ignored") or ()) + pending = [s for s in migration_space.deletions_pending(db, lvol) if s not in ignored] + if pending: + since = params.get("deletion_wait_since") or now + if not params.get("deletion_wait_since"): + task = _record(task, deletion_wait_since=since) + waited = now - since + if waited < constants.REPL_ENDGAME_DELETION_WAIT_MAX_SEC: + xfer_timing.stamp("round_waits_deletion", lvol=lvol_id, round=nxt, + n=len(pending)) + return (f"round {nxt} snapshot waits for {len(pending)} deletion(s) of " + f"the volume's snapshots to complete " + f"({', '.join(p[:8] for p in pending)})"), task + logger.warning("cutover for lvol %s: snapshot deletion(s) %s still pending after " + "%ds; no longer waiting for them before round %d", + lvol_id, ", ".join(pending), int(waited), nxt) + task = _record(task, deletion_ignored=sorted(ignored | set(pending))) + task = drop_params(task, "deletion_wait_since") or task + elif params.get("deletion_wait_since"): + logger.info("cutover for lvol %s: snapshot deletions completed after %ds; " + "taking round %d", lvol_id, int(now - params["deletion_wait_since"]), nxt) + task = drop_params(task, "deletion_wait_since") or task + params = task.function_params + + # ---- rule 3: free space on the source cluster ------------------------ # + cluster_id = migration_space.source_cluster_id(db, lvol, task.cluster_id) + unknown_since = params.get("space_unknown_since", 0) or 0 + why, new_unknown = migration_space.space_hold(db, lvol, cluster_id, unknown_since, now) + if new_unknown != unknown_since: + task = (_record(task, space_unknown_since=new_unknown) if new_unknown + else drop_params(task, "space_unknown_since") or task) + if why: + if not params.get("space_low_since"): + logger.warning("cutover for lvol %s: %s", lvol_id, why) + migration_space.space_low_event(lvol, why) + task = _record(task, space_low_since=now) + xfer_timing.stamp("round_waits_space", lvol=lvol_id, round=nxt) + return why, task + if params.get("space_low_since"): + logger.info("cutover for lvol %s: source space recovered after %ds; taking " + "round %d", lvol_id, int(now - params["space_low_since"]), nxt) + task = drop_params(task, "space_low_since") or task + return "", task + + def _take_round(task, lvol): """Take the next round's snapshot and, with direct rounds, send it into the cutover clone; (error, task). Direct or not is decided once per endgame, at its first round: a clone holding direct rounds is never - rebased onto ordinary rounds' copies.""" + rebased onto ordinary rounds' copies. + + The snapshot waits (an error, so the caller defers) while the volume's + earlier deletions are pending or the source is short of space + (_round_snapshot_hold).""" + held, task = _round_snapshot_hold(task, lvol) + if held: + return held, task params = task.function_params if "direct_rounds" not in params: direct = _direct_mode_possible(lvol, params) @@ -1818,7 +1999,13 @@ def _shrink_step(task, lvol, tasks=()): continue started_at = params.get("shrink_started_at") - if started_at is None: + times = params.get("shrink_round_times") or [] + if params.get("shrink_round_measured") == params["shrink_round"] and times: + # Measured on an earlier pass, which then could not take the next + # snapshot (_round_snapshot_hold, a refused snapshot): the round's + # duration is what was measured then, not the time since. + elapsed = float(times[-1]) + elif started_at is None: # Unmeasurable round (an older task, or one enqueued without the # stamp). Treat it as NOT converged rather than as instant: a # missing measurement must never be read as "the delta is small", @@ -1843,7 +2030,8 @@ def _shrink_step(task, lvol, tasks=()): round=params["shrink_round"], ms=elapsed * 1000.0) task = _record(task, shrink_round_times=round_times, - shrink_round_done_at=time.time()) + shrink_round_done_at=time.time(), + shrink_round_measured=params["shrink_round"]) params = task.function_params # Every round is an endgame round now -- the lvstore is held before the @@ -1973,7 +2161,11 @@ def _final_delta_estimate(params): _ENDGAME_ROUND_PARAMS = ("shrink_snap_id", "shrink_round", "shrink_round_times", "shrink_started_at", "shrink_round_done_at", "direct_rounds", "direct_xfer_node", "direct_submits", - "direct_done_snap", "direct_round_ids") + "direct_done_snap", "direct_round_ids", + # _round_snapshot_hold's waits (bf29) + "shrink_round_measured", + "deletion_wait_since", "deletion_ignored", + "space_low_since", "space_unknown_since") def _abandon_endgame(task, lvol, reason) -> NoReturn: diff --git a/simplyblock_core/test/conftest.py b/simplyblock_core/test/conftest.py index 773897acd2..d87cf7734c 100644 --- a/simplyblock_core/test/conftest.py +++ b/simplyblock_core/test/conftest.py @@ -84,3 +84,15 @@ def ns_host_guard_permits(monkeypatch): the runner tests that want a stranded host patch removal_blocked.""" from simplyblock_core.controllers import ns_host_guard monkeypatch.setattr(ns_host_guard, "removal_blocked", lambda *a, **kw: None) + + +@pytest.fixture(autouse=True) +def migration_space_permits(monkeypatch): + """Migration snapshots wait for free space and for the volume's pending + deletions (simplyblock_core/migration_space.py). The fakes here model no + capacity records and no snapshot table, so neither ever holds; + test_migration_space_bf29 exercises both with the real functions.""" + from simplyblock_core import migration_space + monkeypatch.setattr(migration_space, "space_hold", + lambda *a, **kw: ("", 0)) + monkeypatch.setattr(migration_space, "deletions_pending", lambda *a, **kw: []) diff --git a/simplyblock_core/test/test_migration_space_bf29.py b/simplyblock_core/test/test_migration_space_bf29.py new file mode 100644 index 0000000000..37882d28b0 --- /dev/null +++ b/simplyblock_core/test/test_migration_space_bf29.py @@ -0,0 +1,452 @@ +"""A migration must never fill its SOURCE cluster by itself (bed bf29). + +Bed bf29 (2026-11-20): two 5-node clusters, 8 client volumes under sustained +fio overwrite, a cluster-scope migration A -> B. Five volumes ran their +cutover endgame at once; each shrink round snapshotted its volume every +15-30 s, every snapshot pinned what fio overwrote after it, and the +superseded snapshots were deleted more slowly than new ones were taken. +Cluster A went 41% -> 82% -> 99% in 16 minutes and ran out of space. + +Three rules (simplyblock_core/migration_space.py): + +1. at most REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER endgames per source cluster; + the others keep replicating on their ordinary cadence; +2. no next round snapshot while the volume's earlier deletions are pending; +3. no migration snapshot (round or cadence) while the source's free space is + under the headroom estimate; an unreadable capacity holds only briefly. + +None of them touches client IO: only the migration's own snapshots wait. +""" +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from simplyblock_core import constants, migration_space +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_monitor +from simplyblock_core.services import tasks_runner_replication_final as runner +from simplyblock_core.services.task_runner_base import TaskDefer + +# The real functions: conftest neutralises both for every other test. +REAL_SPACE_HOLD = migration_space.space_hold +REAL_DELETIONS_PENDING = migration_space.deletions_pending + +GiB = 1 << 30 +NOW = 1_000_000.0 + + +@pytest.fixture +def real_rules(monkeypatch): + monkeypatch.setattr(migration_space, "space_hold", REAL_SPACE_HOLD) + monkeypatch.setattr(migration_space, "deletions_pending", REAL_DELETIONS_PENDING) + + +# --------------------------------------------------------------------------- # +# fakes +# --------------------------------------------------------------------------- # + +def _task(uuid, lvol_id, cluster="A", **params): + t = JobSchedule() + t.uuid = uuid + t.cluster_id = cluster + t.function_name = JobSchedule.FN_REPLICATION_FINAL + t.status = JobSchedule.STATUS_RUNNING + t.max_retry = 8 + t.date = 0 + t.function_params = {"lvol_id": lvol_id, **params} + return t + + +def _lvol(uuid, lvs, node="N-A", size=10 * GiB, mode="migration"): + lv = MagicMock() + lv.get_id.return_value = uuid + lv.uuid = uuid + lv.lvs_name = lvs + lv.node_id = node + lv.size = size + lv.top_bdev = f"LVS/{uuid}" + lv.replication_policy_id = "" + lv.replication_mode = mode + lv.pool_uuid = "P" + return lv + + +def _snap(snap_id, status=SnapShot.STATUS_ONLINE): + return SimpleNamespace(get_id=lambda: snap_id, status=status) + + +class _DB: + """The DB reads the rules make: nodes, tasks, capacity, snapshots.""" + + def __init__(self, tasks=(), free=None, total=1000 * GiB, record_age=5.0, + snaps=None, dirty=None, nodes=None): + self.tasks = list(tasks) + self.total = total + self.free = free + self.record_age = record_age + self.snaps = snaps or {} + self.dirty = dirty + self.rpc_calls = 0 + self.now = NOW # when the latest record was written + age + self.nodes = nodes or {"N-A": self._node("N-A", "A"), "N-B": self._node("N-B", "B")} + + def _node(self, node_id, cluster): + def info(bdev): + self.rpc_calls += 1 + if self.dirty is None: + raise RuntimeError("no count") + return {"tracking": True, "complete": True, "dirty_bytes": self.dirty} + rpc = SimpleNamespace(bdev_lvol_dirty_bitmap_info=info) + return SimpleNamespace(get_id=lambda: node_id, cluster_id=cluster, + status=StorageNode.STATUS_ONLINE, + secondary_node_id="", tertiary_node_id="", + rpc_client=lambda **kw: rpc) + + def get_storage_node_by_id(self, node_id): + return self.nodes[node_id] + + def get_job_tasks(self, cluster_id, **kw): + return [t for t in self.tasks if t.cluster_id == cluster_id] + + def get_cluster_by_id(self, cluster_id): + return SimpleNamespace(get_id=lambda: cluster_id) + + def get_cluster_capacity(self, cluster, limit=1): + if self.free is None: + return [] + return [SimpleNamespace(size_total=self.total, size_used=self.total - self.free, + date=int(self.now - self.record_age))] + + def get_snapshots_by_lvol_id(self, lvol_id): + return list(self.snaps.get(lvol_id, ())) + + +@pytest.fixture +def clock(monkeypatch): + now = {"t": NOW} + monkeypatch.setattr(runner.time, "time", lambda: now["t"]) + return now + + +def _use(monkeypatch, fake): + monkeypatch.setattr(runner, "db", fake) + monkeypatch.setattr(snapshot_monitor, "db", fake) + + +# --------------------------------------------------------------------------- # +# Rule 1: one endgame per source cluster +# --------------------------------------------------------------------------- # + +def test_bf29_a_second_lvstore_of_the_same_cluster_queues_for_the_endgame_slot(monkeypatch, clock): + """bf29: five endgames on five lvstores of cluster A ran at once. The + per-lvstore claim let them; the cluster slot does not.""" + holder = _task("T1", "LV1", cutover_lvs="LVS_1", cutover_claimed_at=NOW - 60) + mine = _task("T2", "LV2") + _use(monkeypatch, _DB([holder, mine])) + with pytest.raises(TaskDefer, match="endgame slot of source cluster A"): + runner._acquire_lvs_claim(mine.frozen_view(), _lvol("LV2", "LVS_2"), [holder, mine]) + assert "cutover_lvs" not in mine.function_params, "a queued volume holds no lvstore" + + +def test_bf29_the_queued_volume_keeps_its_ordinary_cadence(monkeypatch): + """Waiting for the slot is not an endgame: the cadence snapshots of the + queued volume go on (no RPO gap added by the wait).""" + holder = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_snap_id="S1") + queued = _task("T2", "LV2", shrink_deadline=NOW + 900) + _use(monkeypatch, _DB([holder, queued])) + assert snapshot_monitor._cutover_endgame_lvol_ids("A") == {"LV1"} + + +def test_bf29_an_endgame_of_another_cluster_does_not_take_the_slot(monkeypatch, clock): + other = _task("T1", "LVB", cluster="B", cutover_lvs="LVS_1", cutover_cluster="B") + mine = _task("T2", "LV2") + _use(monkeypatch, _DB([other, mine])) + out = runner._acquire_lvs_claim(mine.frozen_view(), _lvol("LV2", "LVS_2"), [other, mine]) + assert out.function_params["cutover_lvs"] == "LVS_2" + assert out.function_params["cutover_cluster"] == "A" + + +def test_bf29_a_consistency_group_sibling_joins_its_groups_slot(monkeypatch, clock): + holder = _task("T1", "LV1", cutover_lvs="LVS_1", cutover_group="G1", + cutover_claimed_at=NOW - 60) + mine = _task("T2", "LV2") + _use(monkeypatch, _DB([holder, mine])) + monkeypatch.setattr(runner, "_group_id_for_lvol", lambda lv: "G1") + out = runner._acquire_lvs_claim(mine.frozen_view(), _lvol("LV2", "LVS_1"), [holder, mine]) + assert out.function_params["cutover_lvs"] == "LVS_1" + + +def test_bf29_the_limit_is_configurable(monkeypatch, clock): + monkeypatch.setattr(constants, "REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER", 2) + holder = _task("T1", "LV1", cutover_lvs="LVS_1", cutover_claimed_at=NOW - 60) + second = _task("T2", "LV2") + third = _task("T3", "LV3") + fake = _DB([holder, second, third]) + _use(monkeypatch, fake) + runner._acquire_lvs_claim(second.frozen_view(), _lvol("LV2", "LVS_2"), [holder, second, third]) + assert second.function_params["cutover_lvs"] == "LVS_2" + with pytest.raises(TaskDefer, match="2 of 2 in use"): + runner._acquire_lvs_claim(third.frozen_view(), _lvol("LV3", "LVS_3"), + [holder, second, third]) + + +def test_bf29_claims_racing_in_one_pass_leave_exactly_one_endgame(monkeypatch, clock): + """All five caught up in the same runner pass: the pass's task list shows + no claim to any of them. The claim is re-checked against a fresh read + after it is written, and only the first claimant keeps it.""" + tasks = [_task(f"T{i}", f"LV{i}") for i in range(1, 6)] + stale = [JobSchedule().from_dict(t.to_dict()) for t in tasks] # the pass's view + _use(monkeypatch, _DB(tasks)) + winners, queued = [], [] + for i, t in enumerate(tasks, start=1): + clock["t"] = NOW + i + try: + runner._acquire_lvs_claim(t.frozen_view(), _lvol(f"LV{i}", f"LVS_{i}"), stale) + winners.append(t.uuid) + except TaskDefer as e: + assert "lost to a concurrent claim" in str(e) + queued.append(t.uuid) + assert winners == ["T1"] + assert queued == ["T2", "T3", "T4", "T5"] + for t in tasks[1:]: + assert not t.function_params.get("cutover_lvs"), "a loser withdraws its claim" + + +def test_bf29_a_later_claim_never_displaces_a_running_endgame(monkeypatch, clock): + """Ranking is by claim time, not task age: an older task that claims + late must not take the slot from the endgame already running.""" + running = _task("T9", "LV9", cutover_lvs="LVS_9", cutover_claimed_at=NOW - 300) + older = _task("T0", "LV0") # sorts first by id/creation + _use(monkeypatch, _DB([running, older])) + with pytest.raises(TaskDefer): + runner._confirm_cluster_endgame_slot( + runner._record(older.frozen_view(), cutover_lvs="LVS_0", + cutover_cluster="A", cutover_claimed_at=NOW), "", "A") + assert not older.function_params.get("cutover_lvs") + + +def test_bf29_releasing_the_claim_frees_the_slot(monkeypatch, clock): + holder = _task("T1", "LV1", cutover_lvs="LVS_1", cutover_cluster="A", + cutover_claimed_at=NOW - 60) + _use(monkeypatch, _DB([holder])) + monkeypatch.setattr(runner, "_requeue_direct_rounds", lambda t: None) + runner._release_lvs_claim(holder.frozen_view()) + for key in ("cutover_lvs", "cutover_cluster", "cutover_claimed_at"): + assert key not in holder.function_params + + +# --------------------------------------------------------------------------- # +# Rule 2: the next round waits for the volume's pending deletions +# --------------------------------------------------------------------------- # + +def _hold_db(**kw): + kw.setdefault("free", 900 * GiB) + return _DB(**kw) + + +def test_bf29_no_round_snapshot_while_an_earlier_one_is_still_being_deleted( + monkeypatch, clock, real_rules): + fake = _hold_db(snaps={"LV1": [_snap("S1", SnapShot.STATUS_IN_DELETION), + _snap("S2")]}) + _use(monkeypatch, fake) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=3, direct_rounds=False) + taken = MagicMock() + monkeypatch.setattr(runner, "_take_shrink_snapshot", taken) + err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert "round 4 snapshot waits for 1 deletion" in err + taken.assert_not_called() + assert task.function_params["deletion_wait_since"] == NOW + + +def test_bf29_the_round_is_taken_once_the_deletion_completed(monkeypatch, clock, real_rules): + fake = _hold_db(snaps={"LV1": [_snap("S2")]}) + _use(monkeypatch, fake) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=3, direct_rounds=False, + deletion_wait_since=NOW - 20) + taken = MagicMock(return_value=("S3", None, task)) + monkeypatch.setattr(runner, "_take_shrink_snapshot", lambda t, lv, direct=False: taken(t)) + err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert err is None + taken.assert_called_once() + assert "deletion_wait_since" not in task.function_params + + +def test_bf29_a_stuck_deletion_is_set_aside_after_the_bounded_wait(monkeypatch, clock, real_rules): + """A deletion that never completes (no leader, a clone in deletion) must + not stall the endgame on its own; a newer pending deletion is still + waited for.""" + fake = _hold_db(snaps={"LV1": [_snap("STUCK", SnapShot.STATUS_IN_DELETION)]}) + _use(monkeypatch, fake) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=3, direct_rounds=False, + deletion_wait_since=NOW - constants.REPL_ENDGAME_DELETION_WAIT_MAX_SEC) + why, _ = runner._round_snapshot_hold(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert why == "" + assert task.function_params["deletion_ignored"] == ["STUCK"] + fake.snaps["LV1"].append(_snap("S4", SnapShot.STATUS_IN_DELETION)) + why, _ = runner._round_snapshot_hold(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert "S4" in why and "STUCK" not in why + + +def test_bf29_a_soft_deleted_snapshot_is_not_a_pending_deletion(monkeypatch, real_rules): + soft = SimpleNamespace(get_id=lambda: "S1", status=SnapShot.STATUS_ONLINE, deleted=True) + assert migration_space.deletions_pending(_DB(snaps={"LV1": [soft]}), _lvol("LV1", "L")) == [] + + +def test_bf29_a_held_round_is_not_measured_twice(monkeypatch, clock, real_rules): + """The round finished; the next snapshot waits for a deletion. The next + pass must not book the waiting time as another, longer round.""" + fake = _hold_db(snaps={"LV1": [_snap("S0", SnapShot.STATUS_IN_DELETION)]}) + _use(monkeypatch, fake) + monkeypatch.setattr(runner, "_shrink_round_done", lambda snap_id: True) + monkeypatch.setattr(runner, "_take_shrink_snapshot", MagicMock()) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=2, shrink_snap_id="S1", + shrink_started_at=NOW - 10, shrink_deadline=NOW + 900, + shrink_round_times=[20.0], direct_rounds=False) + lvol = _lvol("LV1", "LVS_1") + with patch.object(runner, "_throttle_source", lambda t, lv: t): + with pytest.raises(TaskDefer, match="waits for 1 deletion"): + runner._shrink_step(task.frozen_view(), lvol) + assert task.function_params["shrink_round_times"] == [20.0, 10.0] + clock["t"] += 60 + with pytest.raises(TaskDefer, match="waits for 1 deletion"): + runner._shrink_step(task.frozen_view(), lvol) + assert task.function_params["shrink_round_times"] == [20.0, 10.0] + runner._take_shrink_snapshot.assert_not_called() + + +def test_bf29_the_endgame_is_not_entered_while_its_first_round_would_wait( + monkeypatch, clock, real_rules): + """Holding the lvstore and the cluster slot through the wait would stop + the other volumes' transfers for nothing.""" + fake = _hold_db(snaps={"LV1": [_snap("S0", SnapShot.STATUS_IN_DELETION)]}) + _use(monkeypatch, fake) + held, _ = runner._round_snapshot_hold(_task("T1", "LV1").frozen_view(), _lvol("LV1", "L")) + assert held + import inspect + src = inspect.getsource(runner.task_runner) + assert src.index("_round_snapshot_hold") < src.index("_acquire_lvs_claim") + + +# --------------------------------------------------------------------------- # +# Rule 3: free space on the source cluster +# --------------------------------------------------------------------------- # + +def test_bf29_no_round_snapshot_with_the_source_nearly_full(monkeypatch, clock, real_rules): + """82% -> 99%: under the 5% floor no migration snapshot is taken, and an + event says why -- once per wait, not once per pass.""" + fake = _hold_db(free=10 * GiB) # 99% used of 1000 GiB + _use(monkeypatch, fake) + events = [] + monkeypatch.setattr(migration_space, "space_low_event", + lambda lv, why: events.append(why)) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=5, direct_rounds=False) + monkeypatch.setattr(runner, "_take_shrink_snapshot", MagicMock()) + err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert "deferred" in err and "floor" in err + err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert err + assert len(events) == 1 + runner._take_shrink_snapshot.assert_not_called() + assert fake.rpc_calls == 0, "under the floor there is nothing to ask SPDK" + + +def test_bf29_space_recovered_lets_the_round_go(monkeypatch, clock, real_rules): + fake = _hold_db(free=900 * GiB) + _use(monkeypatch, fake) + task = _task("T1", "LV1", cutover_lvs="LVS_1", shrink_round=5, direct_rounds=False, + space_low_since=NOW - 120) + monkeypatch.setattr(runner, "_take_shrink_snapshot", + lambda t, lv, direct=False: ("S6", None, t)) + err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) + assert err is None + assert "space_low_since" not in task.function_params + + +def test_bf29_headroom_is_twice_the_bytes_written_since_the_last_snapshot(monkeypatch): + lvol = _lvol("LV1", "L", size=400 * GiB) + # 100 GiB free (> 5% floor of 50 GiB); 60 GiB dirty needs 120 GiB + fake = _DB(free=100 * GiB, dirty=60 * GiB) + verdict, why = migration_space.assess(fake, lvol, "A", now=NOW) + assert verdict == "low" and "120 GiB headroom" in why + fake.dirty = 30 * GiB # needs 60 GiB: fits + assert migration_space.assess(fake, lvol, "A", now=NOW)[0] == "ok" + + +def test_bf29_a_small_volume_on_a_roomy_cluster_asks_spdk_nothing(monkeypatch): + fake = _DB(free=500 * GiB, dirty=1 * GiB) + assert migration_space.assess(fake, _lvol("LV1", "L", size=10 * GiB), "A", now=NOW)[0] == "ok" + assert fake.rpc_calls == 0 + + +def test_bf29_an_unknown_dirty_count_assumes_the_whole_volume(monkeypatch): + fake = _DB(free=100 * GiB, dirty=None) # RPC fails + verdict, _ = migration_space.assess(fake, _lvol("LV1", "L", size=60 * GiB), "A", now=NOW) + assert verdict == "low" # 2 x 60 GiB > 100 GiB + + +def test_bf29_unreadable_capacity_holds_only_for_a_bounded_time(monkeypatch, real_rules): + """Fail-safe: no capacity record (collector down) holds the snapshot for + REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC, then lets it through.""" + fake = _DB(free=None) + lvol = _lvol("LV1", "L") + why, since = migration_space.space_hold(fake, lvol, "A", 0, now=NOW) + assert why and since == NOW + why, since = migration_space.space_hold(fake, lvol, "A", since, now=NOW + 60) + assert why and since == NOW + why, since = migration_space.space_hold( + fake, lvol, "A", since, now=NOW + constants.REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC) + assert why == "" and since == NOW + + +def test_bf29_a_stale_capacity_record_is_no_reading(monkeypatch): + fake = _DB(free=900 * GiB, record_age=constants.REPL_MIGRATION_CAPACITY_STALE_SEC + 1) + verdict, why = migration_space.assess(fake, _lvol("LV1", "L"), "A", now=NOW) + assert verdict == "unknown" and "old" in why + + +def test_bf29_known_low_space_is_never_waited_out(monkeypatch, real_rules): + """Only an UNREADABLE capacity is bounded; a known-full source keeps the + migration snapshot waiting (the endgame's shrink deadline bounds that).""" + fake = _DB(free=1 * GiB) + why, since = migration_space.space_hold(fake, _lvol("LV1", "L"), "A", NOW - 10 ** 6, + now=NOW) + assert why and since == 0 + + +def test_bf29_cadence_of_a_migrating_volume_waits_for_space(monkeypatch, real_rules): + fake = _DB(free=10 * GiB) + _use(monkeypatch, fake) + events = [] + monkeypatch.setattr(migration_space, "space_low_event", + lambda lv, why: events.append(why)) + snapshot_monitor._space_low.clear() + snapshot_monitor._space_unknown_since.clear() + lvol = _lvol("LV1", "L", mode="migration") + assert snapshot_monitor._migration_space_hold(lvol, "A", NOW) + assert snapshot_monitor._migration_space_hold(lvol, "A", NOW + 60) + assert len(events) == 1 + fake.free = 900 * GiB + fake.now = NOW + 120 + assert snapshot_monitor._migration_space_hold(lvol, "A", NOW + 120) == "" + assert "LV1" not in snapshot_monitor._space_low + + +def test_bf29_dr_replication_cadence_is_never_held_for_space(monkeypatch, real_rules): + """The rules bound a migration's own consumption only: a volume that + replicates for DR is not a migration and keeps its cadence.""" + fake = _DB(free=1 * GiB) + _use(monkeypatch, fake) + lvol = _lvol("LV1", "L", mode="replicate") + assert snapshot_monitor._migration_space_hold(lvol, "A", NOW) == "" + + +def test_bf29_client_volume_creation_does_not_consult_the_migration_rules(): + """Capacity thresholds stay alarms: nothing on the client path (volume + create, IO) imports the migration space rules.""" + import inspect + + from simplyblock_core.controllers import lvol_controller + assert "migration_space" not in inspect.getsource(lvol_controller) From d5ed6e0f06a1adc9ec5ddff7b504afc0a7c8e00e Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 14:18:24 +0100 Subject: [PATCH 4/9] fix(replication): an endgame waiting on a down target no longer stops replication Bed bf29 R3b (2026-11-20): the target primary node was shut down during an A -> B migration while one volume was in its cutover endgame. Before the cutover clone exists the endgame waits for the target primary (the clone is built through it), and the wait kept the endgame in place: lvstore claim, round in flight. The endgame takes no snapshot while it waits, and the cadence snapshot skips a volume in its endgame, so nothing replicated the volume: no completed cycle for ~20 min, RPO 1280 s. The wait dates from feb45b996 (2026-08-27), which made it a deferral instead of a burned retry but left the endgame held through it. After REPL_CUTOVER_TARGET_DOWN_GRACE_SEC (60 s) of target outage the endgame is now suspended (_suspend_endgame_for_target): round state dropped, lvstore claim and cluster endgame slot released, source write cap lifted, no attempt burned and no back-off. The volume replicates on its ordinary cadence (still under the migration space rules) until the target is back; then the catch-up gate admits it to a fresh endgame. A shorter blip keeps the endgame. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/constants.py | 6 + .../tasks_runner_replication_final.py | 56 ++++++ .../test/test_cutover_target_down_bf29_r3b.py | 170 ++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 simplyblock_core/test/test_cutover_target_down_bf29_r3b.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index f9094cc9cf..7fb4968129 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -740,6 +740,12 @@ def get_config_var(name, default=None): # goes ahead without the check (logged): a broken stats collector must not # stop migrations for good. REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC = 120 +# A cutover endgame whose target node is down before its clone exists cannot +# progress, and holds its volume out of ordinary replication while it waits +# (bed bf29 R3b: RPO 1280 s). After this long it is given up -- no attempt +# burned, no back-off -- and the volume replicates on its cadence until the +# target is back. +REPL_CUTOVER_TARGET_DOWN_GRACE_SEC = 60 # --- cutover source throttle ---------------------------------------------- # A round converges to overhead / (1 - W/T): with the application writing at # W and the delta transferring at T, no number of rounds gets under the diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index f37487fb29..f2d55248f4 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -754,9 +754,20 @@ def task_runner(task: JobSchedule, tasks): lvol_id, tgt_node.get_id(), tgt_node.status) _record(task, last_error=( f"target node {tgt_node.get_id()[:8]} is {tgt_node.status}")) + # An endgame cannot progress meanwhile, and while it holds the volume + # the volume replicates nothing (bed bf29 R3b): hand it back to the + # ordinary cadence after a short grace (_suspend_endgame_for_target). + if params.get("cutover_lvs") or params.get("shrink_snap_id"): + task = _suspend_endgame_for_target(task, lvol, tgt_node) # Deferred rather than failed: a node being offline is transient, and # burning a retry per pass would exhaust them all before it recovers. raise TaskDefer("target node not online, waiting") + if params.get("target_down_since"): + logger.info("cutover for lvol %s: target node %s is back after %ds", + lvol_id, tgt_node.get_id(), + int(time.time() - params["target_down_since"])) + task = drop_params(task, "target_down_since") or task + params = task.function_params # Cooldown set after a hub-attach failure to give the target node time to # recover and the control plane time to reflect a down state in DB. @@ -1028,6 +1039,51 @@ def task_runner(task: JobSchedule, tasks): _record_cutover_done(task) +def _suspend_endgame_for_target(task, lvol, tgt_node): + """The target node is down before the cutover clone exists: after + REPL_CUTOVER_TARGET_DOWN_GRACE_SEC, give the endgame up -- without + burning an attempt and without a back-off -- and return the task. + + Bed bf29 R3b (2026-11-20): the target primary was shut down while a + volume was in its endgame. The endgame waited for it (the clone is built + through the target primary), and while it held the volume nothing + replicated it: the endgame takes no snapshot while it waits, and the + cadence snapshot and retention skip a volume in its endgame. No cycle + completed for ~20 min (RPO 1280 s). The wait itself dates from feb45b996 + (2026-08-27), which made it a deferral instead of a burned retry, but + left the endgame -- lvstore claim, round in flight -- in place for it. + + Giving it up drops the endgame's round state and releases the lvstore + claim and the cluster's endgame slot, so the volume falls back to its + ordinary cadence (snapshots + delta transfers to whichever target member + leads, still under the migration space rules) and another volume can + take the slot. The source write cap is lifted: the outage is not the + volume's failure to converge, and may last long. The task stays: once the + target is back, the catch-up gate lets it enter a fresh endgame. + """ + params = task.function_params + now = time.time() + since = params.get("target_down_since") + if not since: + return _record(task, target_down_since=now) + if now - since < constants.REPL_CUTOVER_TARGET_DOWN_GRACE_SEC: + return task + reason = (f"target node {tgt_node.get_id()[:8]} {tgt_node.status} for " + f"{int(now - since)}s before the cutover clone exists") + logger.warning("cutover for lvol %s: endgame suspended: %s; the volume replicates " + "on its ordinary cadence until the target is back", + lvol.get_id(), reason) + xfer_timing.stamp("endgame_suspended_target_down", lvol=lvol.get_id(), + round=params.get("shrink_round")) + task = _record(task, last_error=f"endgame suspended: {reason}") + task = _discard_spare_landing(task) or task + task = drop_params(task, *_ENDGAME_ROUND_PARAMS) or task + _release_lvs_claim(task) + _restore_source_throttle(task) + # (the caller defers right away; the claim's release committed on its own) + return task + + def _prepare_early(task, lvol, src_node, tgt_node): """Build the cutover clone as the endgame starts (REPL_CUTOVER_EARLY_PREPARE). diff --git a/simplyblock_core/test/test_cutover_target_down_bf29_r3b.py b/simplyblock_core/test/test_cutover_target_down_bf29_r3b.py new file mode 100644 index 0000000000..915f6f4896 --- /dev/null +++ b/simplyblock_core/test/test_cutover_target_down_bf29_r3b.py @@ -0,0 +1,170 @@ +"""An endgame whose target is down must not stop the volume's replication. + +Bed bf29 round R3b (2026-11-20): the target primary node was shut down +between replication cycles of an A -> B migration. One volume was in its +cutover endgame; the endgame waited for the target primary (the cutover +clone is built through it), and while it held the volume nothing replicated +it -- the endgame takes no snapshot while it waits, and the cadence snapshot +skips a volume in its endgame. No cycle completed for ~20 min, RPO 1280 s. + +After REPL_CUTOVER_TARGET_DOWN_GRACE_SEC the endgame is given up (no attempt +burned, no back-off): round state dropped, lvstore claim and cluster endgame +slot released, source write cap lifted. The volume replicates on its cadence +until the target is back, then enters a fresh endgame. +""" +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from simplyblock_core import constants +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_monitor +from simplyblock_core.services import tasks_runner_replication_final as runner +from simplyblock_core.services.task_runner_base import TaskDefer + +NOW = 2_000_000.0 +GRACE = constants.REPL_CUTOVER_TARGET_DOWN_GRACE_SEC + + +def _node(node_id, status): + return SimpleNamespace(get_id=lambda: node_id, status=status, cluster_id="A") + + +class _DB: + def __init__(self, tasks, tgt_status=StorageNode.STATUS_OFFLINE): + self.tasks = tasks + self.nodes = {"SRC": _node("SRC", StorageNode.STATUS_ONLINE), + "TGT": _node("TGT", tgt_status)} + self.lvol = MagicMock() + self.lvol.get_id.return_value = "LV1" + self.lvol.lvs_name = "LVS_1" + self.lvol.node_id = "SRC" + + def get_lvol_by_id(self, lvol_id): + return self.lvol + + def get_storage_node_by_id(self, node_id): + return self.nodes[node_id] + + def get_job_tasks(self, cluster_id, **kw): + return list(self.tasks) + + +def _endgame_task(**extra): + t = JobSchedule() + t.uuid = "T1" + t.cluster_id = "A" + t.function_name = JobSchedule.FN_REPLICATION_FINAL + t.status = JobSchedule.STATUS_RUNNING + t.retry = 0 + t.max_retry = 8 + t.function_params = {"lvol_id": "LV1", "src_node_id": "SRC", "tgt_node_id": "TGT", + "start_time": int(NOW - 3600), **extra} + return t + + +IN_ENDGAME = dict(cutover_lvs="LVS_1", cutover_cluster="A", cutover_claimed_at=NOW - 300, + shrink_snap_id="S5", shrink_round=5, shrink_round_times=[30.0] * 4, + shrink_started_at=NOW - 20, shrink_deadline=NOW + 600, + direct_rounds=False, throttle_restore=[0, 0, 0, 0]) + + +@pytest.fixture +def env(monkeypatch): + clock = {"t": NOW} + monkeypatch.setattr(runner.time, "time", lambda: clock["t"]) + spies = SimpleNamespace(throttle=MagicMock(), requeue=MagicMock()) + monkeypatch.setattr(runner, "_restore_source_throttle", spies.throttle) + monkeypatch.setattr(runner, "_requeue_direct_rounds", spies.requeue) + monkeypatch.setattr(runner, "_discard_spare_landing", lambda t: t) + + def setup(task, **kw): + fake = _DB([task], **kw) + monkeypatch.setattr(runner, "db", fake) + monkeypatch.setattr(snapshot_monitor, "db", fake) + return fake + return SimpleNamespace(clock=clock, spies=spies, setup=setup) + + +def _pass(task): + with pytest.raises(TaskDefer) as e: + runner.task_runner(task.frozen_view(), [task]) + return str(e.value) + + +def test_bf29_r3b_a_short_target_blip_keeps_the_endgame(env): + task = _endgame_task(**IN_ENDGAME) + env.setup(task) + assert "target node not online" in _pass(task) + assert task.function_params["target_down_since"] == NOW + env.clock["t"] = NOW + GRACE - 1 + _pass(task) + assert task.function_params["cutover_lvs"] == "LVS_1" + assert task.function_params["shrink_snap_id"] == "S5" + env.spies.throttle.assert_not_called() + + +def test_bf29_r3b_a_target_down_past_the_grace_returns_the_volume_to_its_cadence(env): + task = _endgame_task(**IN_ENDGAME) + env.setup(task) + _pass(task) + # the volume is held out of cadence while its endgame holds it ... + assert snapshot_monitor._cutover_endgame_lvol_ids("A") == {"LV1"} + env.clock["t"] = NOW + GRACE + _pass(task) + params = task.function_params + for key in ("cutover_lvs", "cutover_cluster", "cutover_claimed_at", "shrink_snap_id", + "shrink_round", "shrink_round_times"): + assert key not in params, key + # ... and is back on the ordinary cadence once the endgame is suspended + assert snapshot_monitor._cutover_endgame_lvol_ids("A") == set() + assert "endgame suspended" in params["last_error"] + assert task.retry == 0, "a target outage burns no attempt" + assert "endgame_retry_after" not in params, "no back-off: resume when the target is back" + env.spies.throttle.assert_called_once() + env.spies.requeue.assert_called_once() + + +def test_bf29_r3b_the_released_slot_is_free_for_another_volume(env, monkeypatch): + task = _endgame_task(**IN_ENDGAME) + env.setup(task) + _pass(task) + env.clock["t"] = NOW + GRACE + _pass(task) + other = _endgame_task() + other.uuid = "T2" + assert runner._cluster_endgame_slots_taken(other.frozen_view(), [task, other], "", "A") == "" + + +def test_bf29_r3b_a_volume_not_in_its_endgame_is_only_deferred(env): + task = _endgame_task(shrink_deadline=NOW + 900) + env.setup(task) + _pass(task) + env.clock["t"] = NOW + 10 * GRACE + _pass(task) + assert task.function_params["shrink_deadline"] == NOW + 900 + env.spies.throttle.assert_not_called() + env.spies.requeue.assert_not_called() + + +def test_bf29_r3b_the_target_back_clears_the_outage_and_the_endgame_can_resume(env, monkeypatch): + task = _endgame_task(**IN_ENDGAME) + fake = env.setup(task) + _pass(task) + env.clock["t"] = NOW + GRACE + _pass(task) + fake.nodes["TGT"].status = StorageNode.STATUS_ONLINE + + class _Stop(Exception): + pass + + def _lag(lvol): + raise _Stop() + monkeypatch.setattr(runner, "_replication_lag_sec", _lag) + with pytest.raises(_Stop): + # past the target check: the volume is on its way to the catch-up gate + # that admits it to a fresh endgame + runner.task_runner(task.frozen_view(), [task]) + assert "target_down_since" not in task.function_params From 3f90029cdd2843a9318d3c0307118270e49f5827 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 14:18:34 +0100 Subject: [PATCH 5/9] fix(replication): no pre-staged landing volume for a direct endgame round A direct round's snapshot is never replicated (its pages go straight into the cutover clone), so the landing volume _prestage_next_landing created under REP_ was never adopted, and _discard_spare_landing only knows the name of the round after it. Every endgame whose first direct round coincided with a still-running cadence transfer left one full-size internal volume on the target for good (bed bf29 2026-11-20: three REP_repl_round_* volumes on cluster 225e0e10 from rounds 2 and 6). Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/services/snapshot_replication.py | 15 +++++++++++++++ .../test/test_repl_endgame_overhead.py | 11 +++++++++++ 2 files changed, 26 insertions(+) diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 1e5e2c1a01..b27a467ac8 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -656,6 +656,11 @@ def _create_landing_volume(name, snapshot, remote_node, remote_pool_uuid): lvol_priority_class=landing_class) +# = tasks_runner_replication_final.DIRECT_ROUND_PREFIX; a copy, so this module +# does not load the cutover runner (a test keeps the two equal). +_DIRECT_ROUND_PREFIX = "repl_round_" + + def _endgame_next_snap_name(lvol_id, cluster_id): """The name a converging cutover of *lvol_id* gives its NEXT shrink snapshot (next_shrink_snap_name), or "".""" @@ -689,6 +694,16 @@ def _prestage_next_landing(snapshot, remote_lv): next_name = _endgame_next_snap_name(lvol_id, snapshot.cluster_id) if not next_name: return None + # A direct round's snapshot is never replicated: its pages go straight + # into the cutover clone, so nothing ever adopts REP_, and the + # endgame's discard only knows the name of the round after it. Each + # such pre-stage left a full-size internal volume on the target for + # good (bed bf29 2026-11-20: REP_repl_round_572cd23c / 1333e5d1 / + # 41e70a17 on cluster 225e0e10, staged 03:53 / 04:11 / 07:47 by the + # transfer of a cadence snapshot still in flight when the endgame + # entered direct mode, never removed). + if next_name.startswith(_DIRECT_ROUND_PREFIX): + return None rep_name = f"REP_{next_name}" try: db.get_lvol_by_name(rep_name, include_deleted=True) diff --git a/simplyblock_core/test/test_repl_endgame_overhead.py b/simplyblock_core/test/test_repl_endgame_overhead.py index 7914e16ee9..84dbb69786 100644 --- a/simplyblock_core/test/test_repl_endgame_overhead.py +++ b/simplyblock_core/test/test_repl_endgame_overhead.py @@ -290,6 +290,17 @@ def test_a_failure_never_breaks_the_transfer(self): self.create.side_effect = RuntimeError("boom") self.assertIsNone(sr._prestage_next_landing(self.snap, self.remote_lv)) + def test_no_landing_volume_for_a_direct_round(self): + # bf29 (2026-11-20): a direct round is never replicated, so a landing + # volume staged under its name was never adopted nor discarded. + self._names("repl_round_next", "repl_round_next") + self.assertIsNone(sr._prestage_next_landing(self.snap, self.remote_lv)) + self.create.assert_not_called() + + def test_direct_round_prefix_matches_the_cutover_runner(self): + from simplyblock_core.services import tasks_runner_replication_final as trf + self.assertEqual(sr._DIRECT_ROUND_PREFIX, trf.DIRECT_ROUND_PREFIX) + class TestShrinkSnapshotNames(unittest.TestCase): From 754285329c9d6ef68b20da2c426e7ed2508a8e22 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 14:20:47 +0100 Subject: [PATCH 6/9] fix(replication): a replaced round name no longer leaks its pre-staged landing volume Bed bf29 (2026-11-20): three 10 GiB REP_repl_round_* landing volumes stayed on cluster A for good. _take_shrink_snapshot replaces the pre-recorded round name (next_shrink_snap_name) when its kind does not match the round (direct vs ordinary), when a snapshot of that name exists but is not usable, and when the name turns out to be taken; each time the landing volume the replication runner had pre-staged as REP_ was never adopted and nothing reclaimed it. Each of the three now discards it first (_discard_landing_named, the body of _discard_spare_landing taking the name as a parameter): deleted when it exists, is online and no live replication task lands in it. A task table that cannot be read keeps the volume. Co-Authored-By: Claude Opus 5.5 --- .../tasks_runner_replication_final.py | 54 +++++++- .../test/test_round_landing_leak_bf29.py | 126 ++++++++++++++++++ 2 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 simplyblock_core/test/test_round_landing_leak_bf29.py diff --git a/simplyblock_core/services/tasks_runner_replication_final.py b/simplyblock_core/services/tasks_runner_replication_final.py index f2d55248f4..6f9af8c6fd 100644 --- a/simplyblock_core/services/tasks_runner_replication_final.py +++ b/simplyblock_core/services/tasks_runner_replication_final.py @@ -1396,6 +1396,9 @@ def _take_shrink_snapshot(task, lvol, direct=False): round=params.get("shrink_round", 0)) name = params.get("next_shrink_snap_name") or _new_shrink_snap_name(direct) if direct != name.startswith(DIRECT_ROUND_PREFIX): + # The landing volume pre-staged under the old name would never be + # adopted (bf29: three 10 GiB REP_repl_round_* left on cluster A). + _discard_landing_named(name, lvol.get_id(), task.cluster_id) name = _new_shrink_snap_name(direct) # The pre-recorded name may already exist: an earlier pass created the # snapshot but did not get to record it (bf17, 2026-11-08: "Snapshot name @@ -1409,6 +1412,7 @@ def _take_shrink_snapshot(task, lvol, direct=False): new_snap, err = existing[0].get_id(), None else: if existing: + _discard_landing_named(name, lvol.get_id(), task.cluster_id) name = _new_shrink_snap_name(direct) with xfer_timing.phase("take_shrink_snapshot", lvol=lvol.get_id(), round=params.get("shrink_round", 0) + 1): @@ -1416,7 +1420,9 @@ def _take_shrink_snapshot(task, lvol, direct=False): lvol, name, snap_type=SnapShot.TYPE_INTERNAL, **({"replicate": False} if direct else {})) if err and "must be unique" in str(err): - # taken by something not ours: next pass uses a fresh name + # taken by something not ours: next pass uses a fresh name, and + # what was pre-staged for this one is never adopted + _discard_landing_named(name, lvol.get_id(), task.cluster_id) task = _record(task, next_shrink_snap_name=_new_shrink_snap_name(direct)) if err: # Three values like the success path: every caller unpacks three, and @@ -1936,25 +1942,59 @@ def _discard_spare_landing(task): if not name: return task task = drop_params(task, "next_shrink_snap_name") or task + _discard_landing_named(name, (task.function_params or {}).get("lvol_id"), + task.cluster_id) + return task + + +def _landing_in_transfer(landing_id, cluster_id): + """True when a live replication task transfers into *landing_id*, or when + that cannot be ruled out (the task table is unreadable).""" + try: + tasks = db.get_job_tasks(cluster_id) + except Exception as e: # noqa: BLE001 + logger.warning("Cannot read tasks to check landing volume %s: %s", landing_id, e) + return True + for t in tasks: + if (t.function_name == JobSchedule.FN_SNAPSHOT_REPLICATION + and t.status != JobSchedule.STATUS_DONE and not t.canceled + and (t.function_params or {}).get("remote_lvol_id") == landing_id): + return True + return False + + +def _discard_landing_named(name, lvol_id, cluster_id): + """Delete the landing volume pre-staged for round snapshot *name* + (REP_, snapshot_replication._prestage_next_landing) when that name + will not be used: it exists, is online and no transfer lands in it. + + Nothing else reclaims such a volume: bed bf29 (2026-11-20) left three + 10 GiB REP_repl_round_* landing volumes on cluster A for good. Best + effort, logged loudly when it fails.""" + if not name: + return rep_name = f"REP_{name}" try: spare = db.get_lvol_by_name(rep_name) except KeyError: - return task + return except Exception as e: # noqa: BLE001 logger.error("Could not look up the spare landing volume %s: %s", rep_name, e) - return task - if spare.status == LVol.STATUS_IN_DELETION: - return task + return + if spare.status != LVol.STATUS_ONLINE: + return + if _landing_in_transfer(spare.get_id(), cluster_id): + logger.info("cutover for lvol %s: pre-staged landing volume %s is in use by a " + "transfer; kept", lvol_id, rep_name) + return try: from simplyblock_core.controllers import lvol_controller logger.info("cutover for lvol %s: deleting the unused pre-staged landing " - "volume %s", task.function_params.get("lvol_id"), rep_name) + "volume %s", lvol_id, rep_name) lvol_controller.delete_lvol(spare, force_delete=True) except Exception as e: # noqa: BLE001 logger.error("Could not delete the unused pre-staged landing volume %s " "(%s): %s", rep_name, spare.get_id(), e) - return task def _shrink_round_done(snap_id): diff --git a/simplyblock_core/test/test_round_landing_leak_bf29.py b/simplyblock_core/test/test_round_landing_leak_bf29.py new file mode 100644 index 0000000000..4d8094dff6 --- /dev/null +++ b/simplyblock_core/test/test_round_landing_leak_bf29.py @@ -0,0 +1,126 @@ +"""A round name that is replaced takes its pre-staged landing volume with it. + +Bed bf29 (2026-11-20): three 10 GiB REP_repl_round_* landing volumes stayed +on cluster A for good. The cutover names its next round snapshot in advance +(next_shrink_snap_name) and the replication runner pre-stages REP_. +_take_shrink_snapshot replaces that name when its kind does not match the +round (direct vs ordinary), when a snapshot of that name exists but is not +usable, and when the name turns out to be taken -- and left REP_ +behind each time. +""" +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.services import tasks_runner_replication_final as runner + +OLD = "repl_commit_old" +OLD_DIRECT = runner.DIRECT_ROUND_PREFIX + "old" + + +class _DB: + def __init__(self, landings, tasks=()): + self.landings = landings # name -> lvol + self.tasks = list(tasks) + + def get_lvol_by_name(self, name): + return self.landings[name] + + def get_job_tasks(self, cluster_id, **kw): + return list(self.tasks) + + +def _landing(lid, status=LVol.STATUS_ONLINE): + return SimpleNamespace(get_id=lambda: lid, status=status) + + +def _task(name): + t = JobSchedule() + t.uuid = "T1" + t.cluster_id = "A" + t.status = JobSchedule.STATUS_RUNNING + t.function_params = {"lvol_id": "LV1", "shrink_round": 2, "next_shrink_snap_name": name} + return t + + +def _lvol(): + lv = MagicMock() + lv.get_id.return_value = "LV1" + return lv + + +@pytest.fixture +def env(monkeypatch): + deleted = [] + from simplyblock_core.controllers import lvol_controller + monkeypatch.setattr(lvol_controller, "delete_lvol", + lambda lv, force_delete=False: deleted.append(lv.get_id())) + from simplyblock_core.controllers import replication_delta + add = MagicMock(return_value=("SNEW", None)) + monkeypatch.setattr(replication_delta, "add_snapshot", add) + monkeypatch.setattr(runner, "_volume_snapshots", lambda lv: []) + + def use(fake): + monkeypatch.setattr(runner, "db", fake) + return SimpleNamespace(deleted=deleted, add=add, use=use, monkeypatch=monkeypatch) + + +def test_bf29_a_direct_round_drops_the_landing_staged_for_an_ordinary_name(env): + env.use(_DB({f"REP_{OLD}": _landing("L-OLD")})) + task = _task(OLD) + _, err, _ = runner._take_shrink_snapshot(task.frozen_view(), _lvol(), direct=True) + assert err is None + assert env.deleted == ["L-OLD"] + assert env.add.call_args.args[1].startswith(runner.DIRECT_ROUND_PREFIX) + + +def test_bf29_an_ordinary_round_drops_the_landing_staged_for_a_direct_name(env): + env.use(_DB({f"REP_{OLD_DIRECT}": _landing("L-OLD")})) + runner._take_shrink_snapshot(_task(OLD_DIRECT).frozen_view(), _lvol(), direct=False) + assert env.deleted == ["L-OLD"] + + +def test_bf29_a_taken_name_drops_its_landing(env): + env.use(_DB({f"REP_{OLD}": _landing("L-OLD")})) + env.add.return_value = (None, "Snapshot name must be unique") + task = _task(OLD) + _, err, _ = runner._take_shrink_snapshot(task.frozen_view(), _lvol()) + assert "must be unique" in err + assert env.deleted == ["L-OLD"] + assert task.function_params["next_shrink_snap_name"] != OLD + + +def test_bf29_an_unusable_snapshot_of_that_name_drops_its_landing(env): + env.use(_DB({f"REP_{OLD}": _landing("L-OLD")})) + stale = SimpleNamespace(snap_name=OLD, status=SnapShot.STATUS_IN_DELETION) + env.monkeypatch.setattr(runner, "_volume_snapshots", lambda lv: [stale]) + runner._take_shrink_snapshot(_task(OLD).frozen_view(), _lvol()) + assert env.deleted == ["L-OLD"] + + +def test_bf29_a_landing_a_transfer_lands_in_is_kept(env): + busy = JobSchedule() + busy.uuid = "R1" + busy.function_name = JobSchedule.FN_SNAPSHOT_REPLICATION + busy.status = JobSchedule.STATUS_RUNNING + busy.function_params = {"remote_lvol_id": "L-OLD"} + env.use(_DB({f"REP_{OLD}": _landing("L-OLD")}, tasks=[busy])) + runner._take_shrink_snapshot(_task(OLD).frozen_view(), _lvol(), direct=True) + assert env.deleted == [] + + +def test_bf29_a_landing_not_online_is_left_alone(env): + env.use(_DB({f"REP_{OLD}": _landing("L-OLD", LVol.STATUS_IN_DELETION)})) + runner._take_shrink_snapshot(_task(OLD).frozen_view(), _lvol(), direct=True) + assert env.deleted == [] + + +def test_bf29_a_matching_name_keeps_its_landing_for_adoption(env): + env.use(_DB({f"REP_{OLD}": _landing("L-OLD")})) + runner._take_shrink_snapshot(_task(OLD).frozen_view(), _lvol(), direct=False) + assert env.deleted == [] + assert env.add.call_args.args[1] == OLD From c520a213394a66ed64f70818b57ab5b4f5631fa3 Mon Sep 17 00:00:00 2001 From: michael Date: Sat, 21 Nov 2026 07:55:14 +0100 Subject: [PATCH 7/9] fix(leadership): restore the leader of an LVS that is leaderless with every member online bf32 2026-11-21 (R6 tgt-dual-p shutdown, target cluster 70edb423): at 05:16:27.95 a write to LVS_7 (primary 484ff828) and to LVS_10 (primary 9154330e) failed while two other nodes' SPDK were gone, and each primary's SPDK dropped the leadership itself (spdk_lvs_queued_failed_IO). SPDK takes it back only on the next IO; both were replication targets, and replication waits for a leader before it sends. The restarts that followed rebuilt the other members as non-leaders behind the demoted primaries, and nothing restores the leadership of a healthy LVS: 76 min leaderless with every member online, until a manual _recover_leaderless_lvs. - storage_node_ops.healthy_lvs_leadership_state / restore_healthy_lvs_leadership: leaderless-but-healthy = cluster active/degraded, owner online primary, every member ONLINE and answering get_lvstore, none leader; then the guarded _recover_leaderless_lvs on the primary (lock+cooldown, no restart/port-allow task, JM quorum, proven-free twice, reload-then-grant). Not on sync-site or two-node arbitrated clusters. - node monitor: per primary, probe every 30 s, act after 60 s confirmed leaderless, recovery on a background thread. - replication: a leaderless-target wait >= 60 s asks that recovery (every 120 s per task); replication still grants nothing itself. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/constants.py | 8 + .../services/snapshot_replication.py | 49 +++- .../services/storage_node_monitor.py | 86 ++++++ simplyblock_core/storage_node_ops.py | 99 +++++++ .../unit/test_healthy_leaderless_recovery.py | 273 ++++++++++++++++++ 5 files changed, 513 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_healthy_leaderless_recovery.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 7fb4968129..86f21912ea 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -931,6 +931,14 @@ def get_config_var(name, default=None): # leader within about a minute of its primary going down (crashed or shut # down); a wait beyond this means no member could be proven safe to lead. REPL_TARGET_LEADERLESS_ALERT_SEC = 180 +# A transfer that has waited this long for a target lvstore whose members are +# ALL online asks the control plane's guarded leaderless recovery +# (storage_node_ops.restore_healthy_lvs_leadership) to give it a leader, then +# at most every REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC. A primary SPDK that +# dropped the leadership itself takes it back only on IO, and a target gets +# none while replication waits (bf32 2026-11-21: LVS_7/LVS_10, 76 min). +REPL_TARGET_LEADERLESS_RECOVER_SEC = 60 +REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC = 120 # Pass interval of the snapshot replication runner while any transfer or # merge is in flight; TASK_EXEC_INTERVAL_SEC when idle. Not sub-second: each # pass reads the task table (see test_transfer_completion_latency). diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index b27a467ac8..77cee5148d 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -1016,6 +1016,10 @@ def process_snap_replicate_start(task, snapshot): # recovery run from here, during a forced shutdown of the target primary, # left LVS_10 with two leaders and corrupted metadata). A leaderless # target is waited out: the transfer stays suspended and the RPO grows. + # A wait that outlasts REPL_TARGET_LEADERLESS_RECOVER_SEC with every + # target member online asks the control plane's guarded recovery + # (_ask_target_leadership_recovery); that recovery, not replication, + # decides whether a grant is safe. remote_lv_node, lead_why = _stable_target_leader(remote_lv) if remote_lv_node is None: # Waiting, not failing: the target's leadership is not this transfer's @@ -1483,8 +1487,10 @@ def _wait_for_target_leader(task, remote_lv, lead_why): IO, so no member promotes itself when its primary goes down; the node monitor's proven takeover gives it a leader, during the down member's restart too while that member's SPDK is down (bf24 2026-11-14, LVS_22: - the transfer waited ~110 s for the crashed primary to come back). The - wait is polled every REPL_RETRY_BACKOFF_BASE_SEC (step 0, whatever the + the transfer waited ~110 s for the crashed primary to come back). With + every member online the wait asks the control plane's guarded recovery + (_ask_target_leadership_recovery) once it lasts + REPL_TARGET_LEADERLESS_RECOVER_SEC. The wait is polled every REPL_RETRY_BACKOFF_BASE_SEC (step 0, whatever the task's earlier retries), costs no retry, and raises a REPLICATION_STALLED event once it lasts REPL_TARGET_LEADERLESS_ALERT_SEC, then at most every REPL_MERGE_ALERT_EVERY_SEC.""" @@ -1507,14 +1513,53 @@ def _wait_for_target_leader(task, remote_lv, lead_why): except Exception as e: # noqa: BLE001 logger.error("Replication task %s: could not log the leaderless-target " "event: %s", task.uuid, e) + _ask_target_leadership_recovery(task, remote_lv, waited, now) _suspend_for_retry(task, msg, backoff=True, count=_switches_exhausted(task), backoff_step=0) +def _ask_target_leadership_recovery(task, remote_lv, waited, now): + """Once a leaderless-target wait has lasted REPL_TARGET_LEADERLESS_RECOVER_SEC, + ask the control plane to restore the target's leadership, at most every + REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC per task. + + Replication still grants nothing itself and never forces: the recovery + (storage_node_ops.restore_healthy_lvs_leadership) acts only when EVERY + member of the target lvstore is online, answers and reports non-leader, + and then through the guarded reload-then-grant every object path uses. + A member that is down or restarting leaves it to the node monitor's + offline takeover and to the restart, as before (bf8 2026-11-02: the + unguarded grant from here during a forced shutdown made two leaders). + bf32 2026-11-21: both target lvstores (LVS_7, LVS_10) lost their leader to + an SPDK self-demotion on a failed write; every member was back online + minutes later, nothing restored the leadership, and the transfers waited + 76 min.""" + params = task.function_params + if waited < constants.REPL_TARGET_LEADERLESS_RECOVER_SEC: + return + tried = params.get("target_recover_tried_at") or 0 + if now - tried < constants.REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC: + return + params["target_recover_tried_at"] = now + try: + from simplyblock_core import storage_node_ops + leader, why = storage_node_ops.restore_healthy_lvs_leadership( + remote_lv.node_id, remote_lv.lvs_name, + reason=f"replication task {task.uuid} waited {waited}s") + except Exception as e: # noqa: BLE001 + logger.error("Replication task %s: leadership recovery of %s failed: %s", + task.uuid, remote_lv.lvs_name, e) + return + logger.info("Replication task %s: leadership recovery of target %s: %s", + task.uuid, remote_lv.lvs_name, why if leader is None + else f"{why} (leader {leader.get_id()})") + + def _target_leader_found(task): """End a leaderless-target wait: log how long it lasted.""" since = task.function_params.pop("target_leaderless_since", None) task.function_params.pop("target_leaderless_alerted_at", None) + task.function_params.pop("target_recover_tried_at", None) if since: logger.info("Replication task %s: target leadership settled after %ds without " "a leader", task.uuid, int(time.time() - since)) diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index 0d73802d63..399629a807 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -1843,6 +1843,84 @@ def _take_over_leaderless_lvstores(snode): lvs_name, leader.get_id(), snode.get_id(), snode.status) +#: lvstore key -> monotonic time of the last healthy-leaderless probe. +_healthy_leaderless_checked: dict[tuple, float] = {} +#: lvstore key -> monotonic time it was first seen leaderless with every +#: member online (cleared by any other observation). +_healthy_leaderless_since: dict[tuple, float] = {} +#: lvstore keys whose recovery runs right now (a background thread each). +_healthy_leaderless_running: set = set() +_healthy_leaderless_lock = threading.Lock() +HEALTHY_LEADERLESS_CHECK_INTERVAL_SEC = 30 +HEALTHY_LEADERLESS_CONFIRM_SEC = 60 + + +def _restore_leadership_if_leaderless(snode): + """Give the own lvstore of ONLINE primary ``snode`` its leader back when it + has none while every member is online and healthy. + + Leadership is placed by the restart, creation and activation flows, and + the offline takeover (_take_over_leaderless_lvstores) covers a DOWN + member; nothing covered a leaderless lvstore whose members are all up. + bf32 2026-11-21 (R6): with two of five nodes' SPDK gone, a write to LVS_7 + and LVS_10 failed and each primary's SPDK dropped the leadership itself + (it takes it back only on the next IO, and both were replication targets + without client IO). The restarts of the other members then rebuilt them + as non-leaders behind the demoted primaries, and the lvstores stayed + leaderless for 76 min with every member online. + + Probed at most every HEALTHY_LEADERLESS_CHECK_INTERVAL_SEC by the + primary's own monitor thread (one lvstore, three get_lvstore RPCs); acted + on only after it was seen leaderless for HEALTHY_LEADERLESS_CONFIRM_SEC, + so a hand-off in flight is not mistaken for a lost leader. The grant is + storage_node_ops.restore_healthy_lvs_leadership (the guarded + reload-then-grant of _recover_leaderless_lvs, which also refuses while a + restart or port-allow task is active on a member), run on a background + thread: it waits out a safety window and must not stall this node's + liveness checks. Not on sync-replication or two-node arbitrated clusters + (healthy_lvs_leadership_state).""" + if not snode.lvstore or snode.is_secondary_node: + return + key = (snode.cluster_id, snode.lvstore) + now = time.monotonic() + if now - _healthy_leaderless_checked.get(key, -1e9) < HEALTHY_LEADERLESS_CHECK_INTERVAL_SEC: + return + _healthy_leaderless_checked[key] = now + with _healthy_leaderless_lock: + if key in _healthy_leaderless_running: + return + state, _, why = storage_node_ops.healthy_lvs_leadership_state(snode.get_id()) + if state != "leaderless": + _healthy_leaderless_since.pop(key, None) + return + since = _healthy_leaderless_since.setdefault(key, now) + if now - since < HEALTHY_LEADERLESS_CONFIRM_SEC: + logger.warning("%s; restoring its leadership if it stays so for %ss", + why, HEALTHY_LEADERLESS_CONFIRM_SEC) + return + with _healthy_leaderless_lock: + if key in _healthy_leaderless_running: + return + _healthy_leaderless_running.add(key) + + def _run(): + try: + leader, result = storage_node_ops.restore_healthy_lvs_leadership( + snode.get_id(), reason=f"leaderless for {int(time.monotonic() - since)}s, " + f"seen by the node monitor") + logger.info("Leadership recovery of %s: %s", snode.lvstore, result) + if leader is not None: + _healthy_leaderless_since.pop(key, None) + except Exception as e: # noqa: BLE001 + logger.error("Leadership recovery of %s failed: %s", snode.lvstore, e) + finally: + with _healthy_leaderless_lock: + _healthy_leaderless_running.discard(key) + + threading.Thread(target=_run, daemon=True, + name=f"lvs-leadership-{snode.lvstore}").start() + + def _ana_failover_rearm(node_id) -> None: """Forget the one-shot marker so the next offline episode re-triggers.""" _ana_failover_applied.discard(node_id) @@ -3041,6 +3119,14 @@ def check_node(snode): ) storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_ONLINE) readmit_devices_after_node_online(snode.get_id()) + return + + if snode.status == StorageNode.STATUS_ONLINE: + try: + _restore_leadership_if_leaderless(snode) + except Exception as e: # noqa: BLE001 + logger.error("Leadership check of %s on %s failed: %s", + snode.lvstore, snode.get_id(), e) def readmit_devices_after_node_online(node_id): diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 7e09b48725..b8c36bf152 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -12638,6 +12638,105 @@ def reconcile_lvs_leadership(cluster_id, nodes, skip_jm_vuids=(), confirmed=(), return restored, leaderless +def healthy_lvs_leadership_state(owner_id, lvs_name=None): + """``(state, members, why)`` for the LVS of ``owner_id`` (its primary). + + ``state`` is ``"leaderless"`` only when the LVS is leaderless but HEALTHY: + the cluster is active or degraded, the owner is an ONLINE primary whose + health check has not failed and whose lvstore is not in a restart window, + every member is ONLINE, every member answers ``get_lvstore``, and none + reports leadership. ``members`` (primary first) is then the set to recover + over. Otherwise ``"led"`` (``members``: the members reporting leadership) + or ``"skip"`` + (``why`` says what does not hold): an offline or restarting member is the + node monitor's offline takeover and the restart's business, a member that + cannot be asked is a POSSIBLE leader. + + Not on sync-replication clusters (the passive triplet must stay + leaderless; the active one is the promote's) nor on two-node arbitrated + ones (the arbiter reconciles there: reconcile_lvs_leadership). + + ``lvs_name``: the LVS the caller means; when ``owner_id`` is not its + primary (a volume's recorded node), the primary is looked up by name.""" + db = DBController() + try: + owner = db.get_storage_node_by_id(owner_id) + except KeyError: + return "skip", [], f"node {owner_id} unknown" + if lvs_name and owner.lvstore != lvs_name: + owner = next((n for n in db.get_storage_nodes_by_cluster_id(owner.cluster_id) + if n.lvstore == lvs_name and not n.is_secondary_node), None) + if owner is None: + return "skip", [], f"no primary of {lvs_name} found" + lvs_name = owner.lvstore + if not lvs_name or owner.is_secondary_node or not owner.jm_vuid: + return "skip", [], f"{owner.get_id()} holds no primary lvstore" + if _sync_site(owner): + return "skip", [], f"{lvs_name}: sync-replication cluster" + if _cluster_two_node_arbitrated(owner.cluster_id, db=db): + return "skip", [], f"{lvs_name}: two-node arbitrated cluster (the arbiter reconciles)" + cluster = db.get_cluster_by_id(owner.cluster_id) + if cluster.status not in (Cluster.STATUS_ACTIVE, Cluster.STATUS_DEGRADED): + return "skip", [], f"{lvs_name}: cluster is {cluster.status}" + if owner.health_check is False or owner.lvstore_status == "in_creation": + return "skip", [], (f"{lvs_name}: primary {owner.get_id()} health_check=" + f"{owner.health_check} lvstore_status={owner.lvstore_status}") + members = [owner] + for nid in (owner.secondary_node_id, owner.tertiary_node_id): + if not nid: + continue + try: + members.append(db.get_storage_node_by_id(nid)) + except KeyError: + return "skip", [], f"{lvs_name}: member {nid} unknown" + not_online = [f"{m.get_id()[:8]} {m.status}" for m in members + if m.status != StorageNode.STATUS_ONLINE] + if not_online: + return "skip", [], f"{lvs_name}: member(s) not online: {', '.join(not_online)}" + leaders, unknown = _lvs_leader_probe(members, lvs_name) + if leaders: + return "led", leaders, f"{lvs_name} led by {leaders[0].get_id()}" + if unknown: + return "skip", [], f"{lvs_name}: a member could not be asked for its leadership" + return "leaderless", members, f"{lvs_name}: no member of {len(members)} reports leadership" + + +def restore_healthy_lvs_leadership(owner_id, lvs_name=None, reason=""): + """Give a leader back to an LVS that is leaderless while all its members + are online and healthy (healthy_lvs_leadership_state); ``(leader, why)``. + + bf32 2026-11-21 (R6, target LVS_7 and LVS_10): with two of five nodes' + SPDK gone, a write to each LVS failed and its primary SPDK dropped the + leadership itself (spdk_lvs_queued_failed_IO, "Leadership state changed + internally to false"). It takes it back only on the next IO it receives, + and a replication target receives none -- replication waits for a leader + before it sends. The restarts that followed rebuilt the other members as + non-leaders "behind" the demoted primaries without asking whether they + led, and nothing else restores the leadership of a healthy LVS: both + stayed leaderless for 76 min with every member online. + + The grant is the guarded leaderless recovery the object paths use + (_recover_leaderless_lvs: single-flight lock with cooldown, no restart / + port-allow task on a member, JM quorum on the taker, every other member + proven non-leader twice, blob-md reload before set_leader, grant verified), + with the primary as the taker. Nothing is granted when any member is not + online or cannot be asked.""" + state, members, why = healthy_lvs_leadership_state(owner_id, lvs_name) + if state == "led": + return members[0], why + if state != "leaderless": + return None, why + owner = members[0] + logger.warning("LVS %s is leaderless with every member online (%s)%s -- restoring " + "leadership on its primary %s through the guarded leaderless recovery", + owner.lvstore, ", ".join(m.get_id()[:8] for m in members), + f" [{reason}]" if reason else "", owner.get_id()) + taker = _recover_leaderless_lvs(owner.cluster_id, members, owner.lvstore, owner) + if taker is None: + return None, f"{owner.lvstore}: guarded leaderless recovery granted nothing (see its log)" + return taker, f"{owner.lvstore}: leadership restored on {taker.get_id()}" + + def resync_solo_lvs_on_leaders(snode): """After a peer returns: start the journal resync of every LVS its surviving peer ran alone, on that LVS's LEADER only. diff --git a/tests/unit/test_healthy_leaderless_recovery.py b/tests/unit/test_healthy_leaderless_recovery.py new file mode 100644 index 0000000000..d5e2d14c5a --- /dev/null +++ b/tests/unit/test_healthy_leaderless_recovery.py @@ -0,0 +1,273 @@ +"""A leaderless lvstore whose members are ALL online gets its leader back. + +bf32 2026-11-21, round R6 "tgt-dual-p shutdown" (target cluster 70edb423): +the primary and secondary of LVS_7 were shut down and restarted. While the +secondary's restart ran, the SPDK of two other nodes (8de868b9, 2aa1550c) went +away; at 05:16:27.95 a write to LVS_7 on its primary 484ff828 and one to +LVS_10 on its primary 9154330e failed, and each SPDK dropped the leadership +itself ("Queued failed IO and Leadership state changed internally to false +for lvs primary"). SPDK takes it back only on the next IO it receives, and +both lvstores were replication targets: replication waits for a leader before +it sends. The restarts of 8de868b9 and 2aa1550c then rebuilt LVS_10 as +non-leaders "behind" 9154330e without asking whether it led, nobody touched +LVS_7 again, and both stayed leaderless with every member online until a +manual _recover_leaderless_lvs at 06:34 (76 min). + +Now the node monitor (for the primary's own lvstore, after a confirmation +window) and a replication transfer waiting on such a target ask +storage_node_ops.restore_healthy_lvs_leadership, which grants only through the +guarded _recover_leaderless_lvs and only while every member is online and +answers non-leader. +""" +import types +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants, storage_node_ops +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_replication as sr +from simplyblock_core.services import storage_node_monitor as mon + + +def _node(nid, lvstore="", sec="", tert="", leader=False, status=StorageNode.STATUS_ONLINE): + n = MagicMock() + n.get_id.return_value = nid + n.status, n.lvstore, n.cluster_id = status, lvstore, "CL" + n.secondary_node_id, n.tertiary_node_id = sec, tert + n.is_secondary_node, n.jm_vuid = False, 7 if lvstore else 0 + n.health_check, n.lvstore_status, n.site = True, "ready", "" + n.leader = leader + n.rpc_client.return_value.get_lvstore.side_effect = ( + lambda name, _n=n: {"name": name, "lvs leadership": _n.leader}) + return n + + +class _Base(unittest.TestCase): + + def setUp(self): + # LVS_7: primary P (484ff828), secondary S (d10f59c9), tertiary T (9154330e) + self.p = _node("P", "LVS_7", "S", "T") + self.s = _node("S", "LVS_4", "T", "X") + self.t = _node("T", "LVS_10", "X", "Y") + self.nodes = {n.get_id(): n for n in (self.p, self.s, self.t)} + self.cluster = types.SimpleNamespace(status=Cluster.STATUS_ACTIVE) + db = MagicMock() + db.get_storage_node_by_id.side_effect = lambda i: self.nodes[i] + db.get_storage_nodes_by_cluster_id.return_value = list(self.nodes.values()) + db.get_cluster_by_id.return_value = self.cluster + self.recover = MagicMock(side_effect=lambda cid, members, lvs, taker: taker) + self.ops = patch.multiple( + storage_node_ops, DBController=MagicMock(return_value=db), + _cluster_two_node_arbitrated=MagicMock(return_value=False), + _recover_leaderless_lvs=self.recover) + self.ops.start() + self.addCleanup(self.ops.stop) + + +class TestHealthyLeadershipState(_Base): + + def test_leaderless_with_every_member_online(self): + state, members, why = storage_node_ops.healthy_lvs_leadership_state("P") + self.assertEqual(state, "leaderless") + self.assertEqual([m.get_id() for m in members], ["P", "S", "T"]) + self.assertIn("LVS_7", why) + + def test_led(self): + self.t.leader = True + state, leaders, _ = storage_node_ops.healthy_lvs_leadership_state("P") + self.assertEqual((state, [n.get_id() for n in leaders]), ("led", ["T"])) + + def test_a_member_not_online_is_not_ours(self): + for st in (StorageNode.STATUS_OFFLINE, StorageNode.STATUS_RESTARTING, + StorageNode.STATUS_DOWN): + self.s.status = st + state, _, why = storage_node_ops.healthy_lvs_leadership_state("P") + self.assertEqual(state, "skip", st) + self.assertIn("not online", why) + + def test_a_member_that_cannot_be_asked_may_lead(self): + self.s.rpc_client.return_value.get_lvstore.side_effect = RuntimeError("timeout") + state, _, why = storage_node_ops.healthy_lvs_leadership_state("P") + self.assertEqual(state, "skip") + self.assertIn("could not be asked", why) + + def test_not_while_the_cluster_is_suspended(self): + self.cluster.status = Cluster.STATUS_SUSPENDED + self.assertEqual(storage_node_ops.healthy_lvs_leadership_state("P")[0], "skip") + + def test_not_in_a_restart_window_or_with_a_failed_health_check(self): + self.p.lvstore_status = "in_creation" + self.assertEqual(storage_node_ops.healthy_lvs_leadership_state("P")[0], "skip") + self.p.lvstore_status, self.p.health_check = "ready", False + self.assertEqual(storage_node_ops.healthy_lvs_leadership_state("P")[0], "skip") + + def test_not_on_sync_or_two_node_clusters(self): + self.p.site = "site-a" + self.assertEqual(storage_node_ops.healthy_lvs_leadership_state("P")[0], "skip") + self.p.site = "" + with patch.object(storage_node_ops, "_cluster_two_node_arbitrated", return_value=True): + self.assertEqual(storage_node_ops.healthy_lvs_leadership_state("P")[0], "skip") + + def test_the_primary_is_found_by_lvstore_name(self): + # a volume's recorded node_id may not be its lvstore's primary + state, members, _ = storage_node_ops.healthy_lvs_leadership_state("S", "LVS_7") + self.assertEqual(state, "leaderless") + self.assertEqual(members[0].get_id(), "P") + + +class TestRestoreHealthyLeadership(_Base): + + def test_guarded_recovery_on_the_primary(self): + leader, why = storage_node_ops.restore_healthy_lvs_leadership("P", reason="test") + self.assertIs(leader, self.p) + args = self.recover.call_args.args + self.assertEqual((args[0], [m.get_id() for m in args[1]], args[2], args[3].get_id()), + ("CL", ["P", "S", "T"], "LVS_7", "P")) + self.assertIn("restored", why) + + def test_nothing_granted_when_led(self): + self.p.leader = True + leader, _ = storage_node_ops.restore_healthy_lvs_leadership("P") + self.assertIs(leader, self.p) + self.recover.assert_not_called() + + def test_nothing_granted_with_a_member_down(self): + self.t.status = StorageNode.STATUS_OFFLINE + leader, why = storage_node_ops.restore_healthy_lvs_leadership("P") + self.assertIsNone(leader) + self.recover.assert_not_called() + + def test_a_declined_recovery_reports_none(self): + self.recover.side_effect = None + self.recover.return_value = None + leader, why = storage_node_ops.restore_healthy_lvs_leadership("P") + self.assertIsNone(leader) + self.assertIn("granted nothing", why) + + +class TestMonitorRestoresLeadership(unittest.TestCase): + + def setUp(self): + for d in (mon._healthy_leaderless_checked, mon._healthy_leaderless_since): + d.clear() + self.addCleanup(d.clear) + mon._healthy_leaderless_running.clear() + self.p = _node("P", "LVS_7", "S", "T") + self.state = MagicMock(return_value=("leaderless", [self.p], "LVS_7: no member reports")) + self.restore = MagicMock(return_value=(self.p, "LVS_7: leadership restored on P")) + pt = patch.multiple(storage_node_ops, healthy_lvs_leadership_state=self.state, + restore_healthy_lvs_leadership=self.restore) + pt.start() + self.addCleanup(pt.stop) + # run the recovery thread inline + pt = patch.object(mon.threading, "Thread", side_effect=lambda target, **kw: types.SimpleNamespace(start=target)) + pt.start() + self.addCleanup(pt.stop) + self.clock = [1000.0] + pt = patch.object(mon.time, "monotonic", side_effect=lambda: self.clock[0]) + pt.start() + self.addCleanup(pt.stop) + + def _tick(self, dt): + self.clock[0] += dt + mon._restore_leadership_if_leaderless(self.p) + + def test_acts_only_after_the_confirmation_window(self): + self._tick(0) + self.restore.assert_not_called() + self._tick(mon.HEALTHY_LEADERLESS_CHECK_INTERVAL_SEC) + self.restore.assert_not_called() + self._tick(mon.HEALTHY_LEADERLESS_CONFIRM_SEC) + self.restore.assert_called_once() + self.assertEqual(self.restore.call_args.args, ("P",)) + self.assertNotIn(("CL", "LVS_7"), mon._healthy_leaderless_since) + + def test_probe_rate_limited(self): + self._tick(0) + self._tick(1) + self.assertEqual(self.state.call_count, 1) + + def test_any_other_observation_restarts_the_window(self): + self._tick(0) + self.state.return_value = ("led", [self.p], "led by P") + self._tick(mon.HEALTHY_LEADERLESS_CHECK_INTERVAL_SEC) + self.state.return_value = ("leaderless", [self.p], "") + self._tick(mon.HEALTHY_LEADERLESS_CONFIRM_SEC) + self.restore.assert_not_called() + + def test_only_for_a_primary(self): + self.p.is_secondary_node = True + self._tick(0) + self.p.is_secondary_node, self.p.lvstore = False, "" + self._tick(100) + self.state.assert_not_called() + + def test_a_failing_recovery_is_contained(self): + self.restore.side_effect = RuntimeError("boom") + self._tick(0) + self._tick(mon.HEALTHY_LEADERLESS_CONFIRM_SEC) + self.assertNotIn(("CL", "LVS_7"), mon._healthy_leaderless_running) + + +class TestReplicationAsksForRecovery(unittest.TestCase): + + def setUp(self): + self.task = types.SimpleNamespace( + uuid="66c78316", retry=0, status="running", function_result="", + function_params={"snapshot_id": "snap-1"}, write_to_db=MagicMock()) + self.remote_lv = types.SimpleNamespace( + node_id="484ff828", nodes=["484ff828", "d10f59c9", "9154330e"], + lvs_name="LVS_7", cluster_id="CL") + db = MagicMock() + db.get_storage_node_by_id.side_effect = ( + lambda i: types.SimpleNamespace(status="online")) + db.get_snapshot_by_id.return_value = types.SimpleNamespace(lvol="src-lvol") + for target, attr, new in ((sr, "db", db), + (sr.lvol_events, "replication_target_leaderless", MagicMock())): + pt = patch.object(target, attr, new) + pt.start() + self.addCleanup(pt.stop) + self.restore = MagicMock(return_value=(None, "declined")) + pt = patch.object(storage_node_ops, "restore_healthy_lvs_leadership", self.restore) + pt.start() + self.addCleanup(pt.stop) + + def _wait(self, age=None): + if age is not None: + self.task.function_params["target_leaderless_since"] -= age + sr._wait_for_target_leader(self.task, self.remote_lv, + "0 members of LVS_7 report leadership") + + def test_not_before_the_recover_delay(self): + self._wait() + self.restore.assert_not_called() + + def test_asks_once_the_wait_is_long_and_rate_limits(self): + self._wait() + self._wait(constants.REPL_TARGET_LEADERLESS_RECOVER_SEC + 1) + self.restore.assert_called_once() + self.assertEqual(self.restore.call_args.args, ("484ff828", "LVS_7")) + self._wait() + self.restore.assert_called_once() # rate-limited + self.task.function_params["target_recover_tried_at"] -= ( + constants.REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC + 1) + self._wait() + self.assertEqual(self.restore.call_count, 2) + self.assertEqual(self.task.status, "suspended") # still waits + + def test_a_failing_recovery_does_not_break_the_wait(self): + self.restore.side_effect = RuntimeError("fdb") + self._wait() + self._wait(constants.REPL_TARGET_LEADERLESS_RECOVER_SEC + 1) + self.assertEqual(self.task.status, "suspended") + + def test_a_leader_clears_the_rate_limit(self): + self._wait() + self._wait(constants.REPL_TARGET_LEADERLESS_RECOVER_SEC + 1) + sr._target_leader_found(self.task) + self.assertNotIn("target_recover_tried_at", self.task.function_params) + + +if __name__ == "__main__": + unittest.main() From 99993d7dbad8e8233a2187f50c66e8c14e4c3054 Mon Sep 17 00:00:00 2001 From: michael Date: Sat, 21 Nov 2026 08:22:49 +0100 Subject: [PATCH 8/9] fix(replication): one task and one run per snapshot; never delete a landing volume under creation bed bf32 (2026-11-21, cluster 70edb423): two replication workers enqueued the same chain ancestor (snapshot 4d20399d) 100 ms apart. _add_task's dedupe is a scan followed by a write, so both passed it and two tasks (5571400c, a6f9b7b7) existed for one snapshot. The worker pool dedupes by task uuid and _dispatch yielded only to a RUNNING sibling, so both ran at once: one created landing volume 5eeb989b while the other took it (in_creation, 2 s old) for a dead attempt's leftover and deleted it. The delete raced the creator's registers on the secondary/tertiary, which kept a stale LVOL_2398 that later crashed SPDK on both nodes. - DBController.ensure_snapshot_replication_task: the task write and a repl_snap_task// claim key in ONE transaction; _add_task creates every snapshot_replication task through it. - Runner: _Workers runs one task per snapshot at a time; _dispatch lets only one live task of a snapshot run (RUNNING, then landing owner, then oldest), looked up via the repl_snapshot_id index. - Half-created landing cleanup only deletes a provably abandoned volume: not being created in this runner (_LANDING_CREATING), no other live task of the snapshot, older than REPL_LANDING_ABANDON_SEC (600 s). After the delete it confirms the bdev is gone on every lvstore member (sync delete of a leftover registration; durable lvol_sync_del task where unconfirmed). Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/constants.py | 7 + .../controllers/tasks_controller.py | 10 + simplyblock_core/db_controller.py | 40 ++ .../services/snapshot_replication.py | 225 ++++++++++- .../test/test_repl_double_run_bf32.py | 370 ++++++++++++++++++ .../test/test_repl_endgame_overhead.py | 4 + .../test_replication_chain_completeness.py | 3 +- .../test_replication_runner_resilience.py | 3 + 8 files changed, 654 insertions(+), 8 deletions(-) create mode 100644 simplyblock_core/test/test_repl_double_run_bf32.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 86f21912ea..82355e7e7e 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -954,6 +954,13 @@ def get_config_var(name, default=None): REPL_RUNNER_WORKERS = int(os.getenv("REPL_RUNNER_WORKERS", "8")) # Pause before a task that returned a failure is dispatched again. REPL_RUNNER_RETRY_PAUSE_SEC = 3.0 +# An in_creation landing volume (REP_*) whose create is not running in this +# runner and that no other live task of its snapshot may be creating is taken +# for a dead attempt's leftover and deleted only after this long. Landing +# creates take up to ~50 s on a loaded target (bed bf13); bed bf32 (2026-11-21) +# deleted one 2 s into its create, which left a stale registration on the +# secondaries. +REPL_LANDING_ABANDON_SEC = int(os.getenv("REPL_LANDING_ABANDON_SEC", "600")) # Requests a replication transfer keeps in flight. A full transfer sends one # whole cluster per request (16 x 2 MiB = 32 MiB in flight); a delta # (allow_partial) sends at most 64 KiB per request, so it needs a deeper window diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index e9c7dad063..d37ad3e9a0 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -421,6 +421,16 @@ def _add_task(function_name, cluster_id, node_id, device_id, task_obj.function_params = function_params task_obj.max_retry = max_retry task_obj.status = JobSchedule.STATUS_NEW + if function_name == JobSchedule.FN_SNAPSHOT_REPLICATION: + # The scan above is a check-then-write: two replication workers + # enqueueing the same chain ancestor 100 ms apart both passed it and + # wrote two tasks for one snapshot (bed bf32 2026-11-21). The write + # itself is the atomic claim. + active_id, created = db.ensure_snapshot_replication_task(task_obj) + if not created: + logger.info(f"Task found, skip adding new task: {active_id}") + return False + return task_obj.uuid task_obj.write_to_db(db.kv_store) if send_to_cluster_log: tasks_events.task_create(task_obj) diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index d8a96c85b2..d70ee0c112 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -923,6 +923,46 @@ def ensure_sync_resync_task(self, task: JobSchedule, lvs_name: str) -> tuple[str return fdb.transactional(DBController._ensure_sync_resync_task_tx)( self, self.kv_store, task, lvs_name, JobSchedule.active_indexes(self.kv_store)) + # repl_snap_task// -> db key of the snapshot's + # replication task. Read and written in the transaction that creates the + # task, so two creators of the same snapshot's task conflict and the retry + # finds the first one's (bed bf32 2026-11-21). + REPL_SNAP_TASK_PREFIX = "repl_snap_task" + + def _repl_snap_task_key(self, cluster_id, snapshot_id): + return f"{self.REPL_SNAP_TASK_PREFIX}/{cluster_id}/{snapshot_id}".encode() + + def _ensure_snapshot_replication_task_tx(self, tr, task, index_list): + claim_key = self._repl_snap_task_key( + task.cluster_id, task.function_params["snapshot_id"]) + raw = tr.get(claim_key).wait() + if raw is not None and raw.present(): + current = BaseModel._read_record(tr, bytes(raw), JobSchedule) + if (current is not None and current.status != JobSchedule.STATUS_DONE + and not current.canceled): + return current.uuid, False + task.updated_at = str(datetime.datetime.now(datetime.UTC)) + self._put_tx(tr, task, index_list) + tr[claim_key] = task.get_db_id().encode() + return task.uuid, True + + def ensure_snapshot_replication_task(self, task: JobSchedule) -> tuple[str, bool]: + """Create ``task`` (an FN_SNAPSHOT_REPLICATION task) unless its + snapshot already has one that is neither done nor canceled -- checked + and written in ONE transaction with the snapshot's claim key, so + concurrent callers never create two tasks for one snapshot. + + bed bf32 (2026-11-21): two replication workers enqueued the same chain + ancestor 100 ms apart; each saw no task in its scan and each wrote + one. The two tasks then ran side by side: one created the landing + volume while the other deleted it as "half-created", leaving a stale + registration on the secondaries that later crashed SPDK. + + Returns ``(uuid of the snapshot's active task, whether it is task)``. + """ + return self._transact(DBController._ensure_snapshot_replication_task_tx, + task, JobSchedule.active_indexes(self.kv_store)) + def _finish_sync_resync_tx(self, tr, task_key, lvs_name, seq_limit, result, duration_sec, task_indexes): task = BaseModel._read_record(tr, task_key, JobSchedule) diff --git a/simplyblock_core/services/snapshot_replication.py b/simplyblock_core/services/snapshot_replication.py index 77cee5148d..cee1f50633 100644 --- a/simplyblock_core/services/snapshot_replication.py +++ b/simplyblock_core/services/snapshot_replication.py @@ -1,8 +1,10 @@ +import errno import logging import threading import time import uuid from concurrent.futures import ThreadPoolExecutor +from datetime import datetime from simplyblock_core import ( constants, @@ -48,6 +50,12 @@ _HUB_LOCKS: dict = {} _HUB_LOCKS_GUARD = threading.Lock() +# Names of the landing volumes this process is creating right now +# (_create_landing_volume). A record wearing one of them is in_creation +# because its create is still running here, not because an attempt died. +_LANDING_CREATING: dict = {} # name -> number of creates in flight +_LANDING_CREATING_GUARD = threading.Lock() + def _hub_lock(source_node_id, hub_node_id): with _HUB_LOCKS_GUARD: @@ -633,7 +641,138 @@ def _await_transfer_completion(task, snapshot, snode): def _create_landing_volume(name, snapshot, remote_node, remote_pool_uuid): - """add_lvol_ha for the landing copy of *snapshot*: (lvol id, error).""" + """add_lvol_ha for the landing copy of *snapshot*: (lvol id, error). + + The name is registered in _LANDING_CREATING while add_lvol_ha runs, so + the half-created-landing cleanup never takes a volume under creation for + a dead attempt's leftover (bed bf32 2026-11-21).""" + with _LANDING_CREATING_GUARD: + _LANDING_CREATING[name] = _LANDING_CREATING.get(name, 0) + 1 + try: + return _create_landing_volume_unguarded( + name, snapshot, remote_node, remote_pool_uuid) + finally: + with _LANDING_CREATING_GUARD: + left = _LANDING_CREATING.get(name, 1) - 1 + if left > 0: + _LANDING_CREATING[name] = left + else: + _LANDING_CREATING.pop(name, None) + + +def _landing_age_sec(lvol, now=None): + """Seconds since *lvol*'s record was created; None when unknown.""" + try: + created = datetime.fromisoformat(str(lvol.create_dt)) + except (TypeError, ValueError): + return None + return ((datetime.now() if now is None else now) - created).total_seconds() + + +def _landing_creation_live(task, existing, rep_name): + """Why the in_creation landing volume *existing* may still be under + creation -- "" only when it is provably abandoned. + + bed bf32 (2026-11-21): a second task of the same snapshot took the + landing volume another worker was creating for a dead attempt's leftover + and deleted it; the delete ran on the leader while the creator was still + registering the volume on the secondary and tertiary, which kept a stale + registration that SPDK later crashed on. A landing volume is abandoned + only when nothing here is creating it, no other live task of the snapshot + may be creating it, and it has been in_creation longer than any create + takes (REPL_LANDING_ABANDON_SEC, which also covers a create in another + process, e.g. an old runner pod during a rolling update).""" + with _LANDING_CREATING_GUARD: + if _LANDING_CREATING.get(rep_name): + return "its create is running in this runner" + try: + siblings = _live_siblings(task) + except Exception as e: # noqa: BLE001 + return f"the task table cannot be read ({e})" + if siblings: + return f"task {siblings[0].uuid} of the same snapshot is live" + age = _landing_age_sec(existing) + if age is not None and age < constants.REPL_LANDING_ABANDON_SEC: + return (f"it is only {int(age)}s old " + f"(abandoned after {constants.REPL_LANDING_ABANDON_SEC}s)") + return "" + + +def _landing_member_nodes(lvol): + """Every node the landing volume may be registered on: its own nodes and + the members of its lvstore (a register can land before the record lists + the node).""" + ids = [] + for nid in [lvol.node_id] + list(lvol.nodes or []): + if nid and nid not in ids: + ids.append(nid) + try: + primary = db.get_storage_node_by_id(lvol.node_id) + for nid in (getattr(primary, "secondary_node_id", ""), + getattr(primary, "tertiary_node_id", "")): + if nid and nid not in ids: + ids.append(nid) + except KeyError: + pass + return ids + + +def _confirm_landing_gone(lvol, member_ids): + """After a half-created landing volume's delete: confirm its bdev is gone + on every member, sync-deleting a registration left behind. Returns the + members where removal is not confirmed; each of them gets a durable + sync-delete task, so no registration is left behind untracked. + + A record still present (in_deletion) is the lvol monitor's to finish; + nothing is probed then.""" + try: + db.get_lvol_by_id(lvol.get_id()) + return [] + except KeyError: + pass + if not lvol.lvol_bdev or not lvol.lvs_name: + return [] + bdev = f"{lvol.lvs_name}/{lvol.lvol_bdev}" + unconfirmed = [] + for nid in member_ids: + try: + node = db.get_storage_node_by_id(nid) + except KeyError: + continue # gone with its lvstore: owes nothing + confirmed = False + if node.status == StorageNode.STATUS_ONLINE: + try: + rpc = node.rpc_client(timeout=5, retry=2) + if rpc.bdev_get(bdev) is None: + confirmed = True + else: + logger.warning("Landing volume %s left %s registered on %s; " + "deleting it there", lvol.get_id(), bdev, nid[:8]) + try: + rpc.delete_lvol(bdev, sync=True) + except RPCException as e: + if getattr(e, "code", None) != -errno.ENODEV: + raise + confirmed = rpc.bdev_get(bdev) is None + except Exception as e: # noqa: BLE001 + logger.warning("Cannot confirm %s is gone on %s: %s", bdev, nid[:8], e) + if not confirmed: + unconfirmed.append(nid) + try: + from simplyblock_core.controllers import tasks_controller + tasks_controller.add_lvol_sync_del_task( + node.cluster_id, nid, bdev, lvol.node_id) + except Exception as e: # noqa: BLE001 + logger.error("Could not queue the delete of %s on %s: %s", + bdev, nid[:8], e) + if unconfirmed: + logger.error("Removal of landing volume %s (%s) not confirmed on %s; " + "a durable sync delete is queued for each", + lvol.get_id(), bdev, [n[:8] for n in unconfirmed]) + return unconfirmed + + +def _create_landing_volume_unguarded(name, snapshot, remote_node, remote_pool_uuid): # Carry the source volume's subsystem-packing capacity onto the landing # copy. Without it the copy defaults to a one-namespace subsystem, which # can never be joined by later namespaced volumes -- so every replicated @@ -961,6 +1100,9 @@ def process_snap_replicate_start(task, snapshot): logger.warning(f"Landing volume {existing.get_id()} ({rep_name}) has size " f"{existing.size}, the snapshot {snapshot.size}; replacing it") existing.status = LVol.STATUS_IN_CREATION # -> the delete branch below + replacing = True + else: + replacing = False if existing is not None: if existing.status == LVol.STATUS_ONLINE: logger.info(f"Adopting landing volume {existing.get_id()} " @@ -973,13 +1115,24 @@ def process_snap_replicate_start(task, snapshot): _suspend_for_retry(task, f"stale landing volume {rep_name} still deleting, retrying") return else: + # Only a provably abandoned one (bed bf32 2026-11-21: the + # volume deleted here was being created by another worker). + busy = "" if replacing else _landing_creation_live(task, existing, rep_name) + if busy: + _suspend_for_retry( + task, f"landing volume {rep_name} is in_creation and may " + f"still be under creation ({busy}); waiting", + count=False, level=logging.INFO) + return logger.warning(f"Deleting half-created landing volume " f"{existing.get_id()} ({rep_name}, status " f"{existing.status}) from an interrupted attempt") + members = _landing_member_nodes(existing) try: lvol_controller.delete_lvol(existing, force_delete=True) except Exception as e: logger.error(f"Failed to clear stale landing volume {rep_name}: {e}") + _confirm_landing_gone(existing, members) _suspend_for_retry(task, f"cleared stale landing volume {rep_name}, retrying") return @@ -3333,15 +3486,62 @@ def _wait(seconds, wake): _check_endgame_wake(wake) +def _live_siblings(task): + """The other live (not done, not canceled) replication tasks of *task*'s + snapshot, through the repl_snapshot_id index (a point read, not a scan + of the never-pruned task table).""" + snapshot_id = (task.function_params or {}).get("snapshot_id") + if not snapshot_id: + return [] + return [t for t in db.get_replication_tasks_for_snapshot(snapshot_id) + if t.uuid != task.uuid + and (not task.cluster_id or t.cluster_id == task.cluster_id) + and t.function_name == JobSchedule.FN_SNAPSHOT_REPLICATION + and t.status != JobSchedule.STATUS_DONE and t.canceled is False] + + +def _task_precedence(t): + """Sort key: which of several live tasks of one snapshot may run. A + transfer already running first, then one that owns a landing volume, + then the oldest; the uuid breaks ties, so every worker picks the same.""" + params = t.function_params or {} + return (t.status != JobSchedule.STATUS_RUNNING, + not params.get("remote_lvol_id"), + int(t.date or 0), t.uuid) + + +def _yields_to_sibling(task): + """The task of the same snapshot *task* must wait for, or None. + + One snapshot has one replication task (_add_task claims it atomically), + but a duplicate written before that claim existed -- or by any other + path -- must not run beside it: two runs of one snapshot each create and + delete the same REP_* landing volume (bed bf32 2026-11-21). Of the live + tasks of a snapshot exactly one runs, by _task_precedence; the others + wait until it is done, then find the snapshot replicated.""" + siblings = _live_siblings(task) + if not siblings: + return None + first = min(siblings + [task], key=_task_precedence) + return None if first.uuid == task.uuid else first + + def _dispatch(task): """Run one replication task the way a pass does; task_runner's result (False on a failure, which the caller follows with a short pause).""" - if task.status in [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED]: - for t in db.get_job_tasks(task.cluster_id): - if t.function_name == JobSchedule.FN_SNAPSHOT_REPLICATION and t.function_params["snapshot_id"] == task.function_params['snapshot_id']: - if t.status == JobSchedule.STATUS_RUNNING and t.canceled is False: - logger.info("replication task found for same snapshot, retry") - return True + if task.status != JobSchedule.STATUS_DONE and not task.canceled: + try: + first = _yields_to_sibling(task) + except Exception as e: # noqa: BLE001 + # Unknown is not "alone": run it on a later pass. + logger.warning("Cannot read the tasks of snapshot %s: %s", + task.function_params.get("snapshot_id"), e) + return False + if first is not None: + logger.info("Replication task %s waits: task %s of the same snapshot %s " + "runs first", task.uuid, first.uuid, + task.function_params.get("snapshot_id")) + return True if task.status == JobSchedule.STATUS_DONE: return True # Re-read the task in case cancel changed it. If it has @@ -3410,6 +3610,7 @@ def __init__(self, size): if self.size else None) self._guard = threading.Lock() self._inflight: dict = {} # task uuid -> Future + self._inflight_snaps: dict = {} # snapshot id -> task uuid running it self._not_before: dict = {} # task uuid -> earliest next submit def paused(self, task): @@ -3427,11 +3628,18 @@ def submit(self, task): result then, True otherwise.""" if self._pool is None: return _dispatch(task) + snap_id = (task.function_params or {}).get("snapshot_id") or "" with self._guard: if task.uuid in self._inflight: return True + # One snapshot on one worker: a second task of the same snapshot + # never runs beside the first (bed bf32 2026-11-21). + if snap_id and self._inflight_snaps.get(snap_id, task.uuid) != task.uuid: + return True if time.time() < self._not_before.get(task.uuid, 0.0): return True + if snap_id: + self._inflight_snaps[snap_id] = task.uuid self._inflight[task.uuid] = self._pool.submit(self._run, task) return True @@ -3445,6 +3653,9 @@ def _run(self, task): finally: with self._guard: self._inflight.pop(task.uuid, None) + snap_id = (task.function_params or {}).get("snapshot_id") or "" + if snap_id and self._inflight_snaps.get(snap_id) == task.uuid: + del self._inflight_snaps[snap_id] if not res: self._not_before[task.uuid] = ( time.time() + constants.REPL_RUNNER_RETRY_PAUSE_SEC) diff --git a/simplyblock_core/test/test_repl_double_run_bf32.py b/simplyblock_core/test/test_repl_double_run_bf32.py new file mode 100644 index 0000000000..def4fae636 --- /dev/null +++ b/simplyblock_core/test/test_repl_double_run_bf32.py @@ -0,0 +1,370 @@ +"""One replication task per snapshot, run by one worker; a half-created +landing volume is deleted only when provably abandoned, and everywhere. + +bed bf32 (2026-11-21, cluster 70edb423): two replication workers enqueued the +same chain ancestor (snapshot 4d20399d) 100 ms apart. Each scanned the task +table, found no task, and wrote one: 5571400c and a6f9b7b7. Both ran on the +worker pool at once -- the pool dedupes by task uuid only, and _dispatch only +yielded to a sibling that was already RUNNING. 5571400c created landing +volume 5eeb989b (REP_repl_internal_7438cbe8_1795231066); 2 s into that create +a6f9b7b7 found the record in_creation, took it for a dead attempt's leftover +and deleted it. The delete ran while the creator was still registering +LVOL_2398 on the secondary and tertiary, so those kept a registration nothing +tracked; SPDK reused the blob id and both nodes crashed 15 min later. +""" +import threading +import time +from datetime import datetime, timedelta +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest + +from simplyblock_core import constants +from simplyblock_core.controllers import tasks_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import snapshot_replication as sr + +SNAP = "4d20399d-acad-4a11-a1e1-7d8ebd9a44d8" +REP_NAME = "REP_repl_internal_7438cbe8_1795231066" + + +class FakeKV: + """Dict-backed store for the transactionless path (see test_ns_slot_txn).""" + + def __init__(self): + self.data = {} + + def get(self, key): + return self.data.get(key) + + def set(self, key, value): + self.data[key] = value + + def clear(self, key): + self.data.pop(key, None) + + def add(self, key, value): + pass + + def get_range_startswith(self, prefix, limit=0, reverse=False): + return sorted((k, v) for k, v in self.data.items() if k.startswith(prefix)) + + +@pytest.fixture +def dbc(): + d = DBController() + saved = d.kv_store + d.kv_store = FakeKV() + yield d + d.kv_store = saved + + +def _repl_task(uuid, snap=SNAP, status=JobSchedule.STATUS_NEW, date=1795237024, **params): + t = JobSchedule() + t.uuid = uuid + t.cluster_id = "70edb423" + t.node_id = "node" + t.date = date + t.function_name = JobSchedule.FN_SNAPSHOT_REPLICATION + t.function_params = {"snapshot_id": snap, "replicate_to_source": True, **params} + t.status = status + t.max_retry = 8 + return t + + +# -- (1) one task per snapshot: the atomic claim -------------------------------- + +def test_bf32_second_task_of_a_snapshot_is_refused_by_the_claim(dbc): + first, created = dbc.ensure_snapshot_replication_task(_repl_task("5571400c")) + assert (first, created) == ("5571400c", True) + other, created = dbc.ensure_snapshot_replication_task(_repl_task("a6f9b7b7")) + assert (other, created) == ("5571400c", False) + assert dbc.get_task_by_id("5571400c").uuid == "5571400c" + with pytest.raises(KeyError): + dbc.get_task_by_id("a6f9b7b7") + + +def test_bf32_a_done_task_frees_the_snapshot_claim(dbc): + t = _repl_task("5571400c") + dbc.ensure_snapshot_replication_task(t) + t.status = JobSchedule.STATUS_DONE + t.write_to_db(dbc.kv_store) + assert dbc.ensure_snapshot_replication_task(_repl_task("a6f9b7b7")) == ("a6f9b7b7", True) + + +def test_bf32_two_concurrent_enqueues_of_one_ancestor_create_one_task(dbc): + """The bf32 interleaving: both enqueuers' scans find nothing. The write + is the claim (one FDB transaction in production; serialized here the way + FDB's conflict retry serializes it), so only one task is created.""" + txn = threading.Lock() + real = dbc.ensure_snapshot_replication_task + + def serialized(task): + with txn: + return real(task) + + stub = MagicMock() + stub.get_job_tasks.return_value = [] # both scans: no task yet + stub.ensure_snapshot_replication_task.side_effect = serialized + barrier = threading.Barrier(2) + results = [] + + def enqueue(): + barrier.wait() + results.append(tasks_controller.add_snapshot_replication_task( + "70edb423", "node", SNAP, replicate_to_source=True)) + + with patch.object(tasks_controller, "db", stub): + threads = [threading.Thread(target=enqueue) for _ in range(2)] + for th in threads: + th.start() + for th in threads: + th.join(5) + created = [r for r in results if r] + assert len(results) == 2 and len(created) == 1, results + stub.kv_store.assert_not_called() + assert sum(1 for k in dbc.kv_store.data if k.startswith(b"repl_snap_task/")) == 1 + + +# -- (1) one run per snapshot: the worker pool and _dispatch ---------------------- + +def _wait_until(cond, timeout=5.0): + end = time.time() + timeout + while time.time() < end: + if cond(): + return True + time.sleep(0.01) + return cond() + + +def test_bf32_two_tasks_of_one_snapshot_never_run_at_once_on_the_pool(): + runs, release = [], threading.Event() + + def slow(task): + runs.append(task.uuid) + release.wait(5) + return True + + pool = sr._Workers(4) + with patch.object(sr, "_dispatch", side_effect=slow): + pool.submit(_repl_task("5571400c")) + assert _wait_until(lambda: runs == ["5571400c"]) + pool.submit(_repl_task("a6f9b7b7")) # same snapshot, other task + time.sleep(0.1) + assert runs == ["5571400c"] + release.set() + assert _wait_until(lambda: pool.running() == 0) + pool.submit(_repl_task("a6f9b7b7")) # the first is done: may run now + assert _wait_until(lambda: runs == ["5571400c", "a6f9b7b7"]) + + +class _TaskDB: + def __init__(self, tasks): + self.tasks = {t.uuid: t for t in tasks} + + def get_replication_tasks_for_snapshot(self, snapshot_id): + return [t for t in self.tasks.values() + if t.function_params.get("snapshot_id") == snapshot_id] + + def get_task_by_id(self, uuid): + return self.tasks[uuid] + + +@pytest.mark.parametrize("status", [JobSchedule.STATUS_NEW, JobSchedule.STATUS_SUSPENDED]) +def test_bf32_only_one_of_two_live_tasks_of_a_snapshot_is_dispatched(status): + """Neither task was RUNNING in bf32 (the landing create comes first), so + the old RUNNING-only yield let both through.""" + older = _repl_task("5571400c", status=status, date=100) + newer = _repl_task("a6f9b7b7", status=status, date=101) + ran = [] + with patch.object(sr, "db", _TaskDB([older, newer])), \ + patch.object(sr, "task_runner", side_effect=lambda t: ran.append(t.uuid) or True): + assert sr._dispatch(newer) is True + assert sr._dispatch(older) is True + assert ran == ["5571400c"] + + +def test_bf32_the_task_owning_a_landing_volume_runs_first(): + older = _repl_task("5571400c", date=100) + newer = _repl_task("a6f9b7b7", date=101, remote_lvol_id="5eeb989b") + ran = [] + with patch.object(sr, "db", _TaskDB([older, newer])), \ + patch.object(sr, "task_runner", side_effect=lambda t: ran.append(t.uuid) or True): + sr._dispatch(older) + sr._dispatch(newer) + assert ran == ["a6f9b7b7"] + + +# -- (2) the half-created landing cleanup ---------------------------------------- + +def _landing(age_sec, status=LVol.STATUS_IN_CREATION): + lv = LVol() + lv.uuid = "5eeb989b-5ca6-4ac0-9f44-4cacde14dbb3" + lv.lvol_name = REP_NAME + lv.status = status + lv.node_id = "9154330e" + lv.nodes = ["9154330e"] # the secondaries not recorded yet + lv.lvs_name = "LVS_10" + lv.lvol_bdev = "LVOL_2398" + lv.create_dt = str(datetime.now() - timedelta(seconds=age_sec)) + return lv + + +def _node(nid, status=StorageNode.STATUS_ONLINE, rpc=None, **kw): + n = SimpleNamespace(cluster_id="70edb423", status=status, + secondary_node_id="", tertiary_node_id="", **kw) + n.get_id = lambda: nid + n.rpc_client = lambda **_: rpc + return n + + +class _StartDB: + """Enough of the DB for process_snap_replicate_start to reach the + landing-volume probe.""" + + def __init__(self, landing, tasks, nodes): + self.landing, self.tasks, self.nodes = landing, tasks, nodes + self.deleted = False + + def get_storage_node_by_id(self, nid): + if nid not in self.nodes: + raise KeyError(nid) + return self.nodes[nid] + + def get_lvol_by_name(self, name, include_deleted=False): + if name != REP_NAME or self.deleted: + raise KeyError(name) + return self.landing + + def get_lvol_by_id(self, lvol_id): + if self.deleted or lvol_id != self.landing.get_id(): + raise KeyError(lvol_id) + return self.landing + + def get_replication_tasks_for_snapshot(self, snapshot_id): + return [t for t in self.tasks if t.function_params.get("snapshot_id") == snapshot_id] + + +def _run_start(task, sdb): + snapshot = SimpleNamespace( + snap_name="repl_internal_7438cbe8_1795231066", source_replicated_snap_uuid="", + target_replicated_snap_uuid="", cluster_id="70edb423", size=10 << 30, + lvol=SimpleNamespace(node_id="SRC", get_id=lambda: "7438cbe8"), + get_id=lambda: SNAP) + delete = MagicMock(side_effect=lambda lv, **kw: setattr(sdb, "deleted", True)) + suspended = [] + with patch.object(sr, "db", sdb), \ + patch.object(sr, "_lvs_transfer_hold", return_value=None), \ + patch.object(sr, "_merge_behind_hold", return_value=None), \ + patch.object(sr, "_select_source_node", + return_value=(SimpleNamespace(get_id=lambda: "SRC", cluster_id="SRCCL"), + "primary")), \ + patch.object(sr, "_unreplicated_local_ancestor", return_value=("ok", None, "")), \ + patch.object(sr, "_destination_pool_uuid", return_value="POOL"), \ + patch.object(sr, "_counterpart_on_destination", return_value=None), \ + patch.object(sr, "_create_landing_volume") as create, \ + patch.object(sr, "_suspend_for_retry", + side_effect=lambda t, msg, **kw: suspended.append(msg)), \ + patch.object(sr.lvol_controller, "delete_lvol", delete), patch.object(JobSchedule, "write_to_db"), \ + patch("simplyblock_core.controllers.tasks_controller.add_lvol_sync_del_task") as queued: + sr.process_snap_replicate_start(task, snapshot) + create.assert_not_called() + return delete, suspended, queued + + +def test_bf32_cleanup_does_not_delete_a_landing_volume_another_task_creates(): + """The exact bf32 state: a6f9b7b7 meets 5eeb989b in_creation, 2 s old, + while 5571400c (live) creates it.""" + me = _repl_task("a6f9b7b7", status=JobSchedule.STATUS_SUSPENDED) + creator = _repl_task("5571400c") + sdb = _StartDB(_landing(age_sec=2), [me, creator], + {"node": _node("node"), "9154330e": _node("9154330e")}) + delete, suspended, _ = _run_start(me, sdb) + delete.assert_not_called() + assert "5571400c" in suspended[0] and "under creation" in suspended[0] + + +def test_bf32_cleanup_does_not_delete_a_landing_volume_this_runner_creates(): + me = _repl_task("a6f9b7b7", status=JobSchedule.STATUS_SUSPENDED) + sdb = _StartDB(_landing(age_sec=constants.REPL_LANDING_ABANDON_SEC * 2), [me], + {"node": _node("node"), "9154330e": _node("9154330e")}) + with sr._LANDING_CREATING_GUARD: + sr._LANDING_CREATING[REP_NAME] = 1 + try: + delete, suspended, _ = _run_start(me, sdb) + finally: + with sr._LANDING_CREATING_GUARD: + sr._LANDING_CREATING.pop(REP_NAME, None) + delete.assert_not_called() + assert "running in this runner" in suspended[0] + + +def test_bf32_cleanup_does_not_delete_a_young_landing_volume(): + """No live sibling and nothing here creating it -- but a create in another + process (a runner pod being replaced) cannot be seen: age decides.""" + me = _repl_task("a6f9b7b7", status=JobSchedule.STATUS_SUSPENDED) + sdb = _StartDB(_landing(age_sec=30), [me], + {"node": _node("node"), "9154330e": _node("9154330e")}) + delete, suspended, _ = _run_start(me, sdb) + delete.assert_not_called() + assert "old" in suspended[0] + + +def test_bf32_a_landing_volume_mid_create_is_registered_while_add_lvol_ha_runs(): + seen = [] + with patch.object(sr, "_create_landing_volume_unguarded", + side_effect=lambda *a: seen.append(dict(sr._LANDING_CREATING)) or ("id", None)): + assert sr._create_landing_volume(REP_NAME, None, None, None) == ("id", None) + assert seen == [{REP_NAME: 1}] + assert REP_NAME not in sr._LANDING_CREATING + + +class _Rpc: + def __init__(self, present): + self.present = present + self.deleted = [] + + def bdev_get(self, name): + return {"name": name} if self.present else None + + def delete_lvol(self, name, sync=False): + self.deleted.append((name, sync)) + self.present = False + return True + + +def test_bf32_cleanup_deletes_an_abandoned_landing_volume_on_the_secondaries_too(): + """Abandoned: deleted -- and confirmed gone on every lvstore member, even + the ones the record did not list yet. The secondary that kept LVOL_2398 + registered gets a sync delete; an offline tertiary a durable task.""" + me = _repl_task("a6f9b7b7", status=JobSchedule.STATUS_SUSPENDED) + prim_rpc, sec_rpc = _Rpc(False), _Rpc(True) + nodes = { + "node": _node("node"), + "9154330e": _node("9154330e", rpc=prim_rpc), + "8de868b9": _node("8de868b9", rpc=sec_rpc), + "2aa1550c": _node("2aa1550c", status=StorageNode.STATUS_OFFLINE), + } + nodes["9154330e"].secondary_node_id = "8de868b9" + nodes["9154330e"].tertiary_node_id = "2aa1550c" + sdb = _StartDB(_landing(age_sec=constants.REPL_LANDING_ABANDON_SEC + 60), [me], nodes) + delete, suspended, queued = _run_start(me, sdb) + delete.assert_called_once() + assert delete.call_args.kwargs == {"force_delete": True} + assert prim_rpc.deleted == [] + assert sec_rpc.deleted == [("LVS_10/LVOL_2398", True)] + queued.assert_called_once_with("70edb423", "2aa1550c", "LVS_10/LVOL_2398", "9154330e") + assert "cleared stale landing volume" in suspended[0] + + +def test_bf32_a_record_still_in_deletion_is_left_to_the_monitor(): + lv = _landing(age_sec=9999, status=LVol.STATUS_IN_DELETION) + rpc = _Rpc(True) + sdb = _StartDB(lv, [], {"9154330e": _node("9154330e", rpc=rpc)}) + with patch.object(sr, "db", sdb): + assert sr._confirm_landing_gone(lv, ["9154330e"]) == [] + assert rpc.deleted == [] diff --git a/simplyblock_core/test/test_repl_endgame_overhead.py b/simplyblock_core/test/test_repl_endgame_overhead.py index 84dbb69786..afe973cf02 100644 --- a/simplyblock_core/test/test_repl_endgame_overhead.py +++ b/simplyblock_core/test/test_repl_endgame_overhead.py @@ -63,6 +63,10 @@ def get_clusters(self): def get_job_tasks(self, cluster_id, reverse=False): return list(self.tasks.values()) + def get_replication_tasks_for_snapshot(self, snapshot_id): + return [t for t in self.tasks.values() + if (t.function_params or {}).get("snapshot_id") == snapshot_id] + def get_task_by_id(self, uuid): if uuid not in self.tasks: raise KeyError(uuid) diff --git a/simplyblock_core/test/test_replication_chain_completeness.py b/simplyblock_core/test/test_replication_chain_completeness.py index 4b0df02ba8..66ef968e11 100644 --- a/simplyblock_core/test/test_replication_chain_completeness.py +++ b/simplyblock_core/test/test_replication_chain_completeness.py @@ -617,7 +617,8 @@ def test_interrupted_landing_volume_is_adopted_or_cleared(): src = inspect.getsource(sr.process_snap_replicate_start) probe = src.index('rep_name = f"REP_{snapshot.snap_name}"') create = src.index("_create_landing_volume(") - assert "lvol_controller.add_lvol_ha" in inspect.getsource(sr._create_landing_volume) + assert "lvol_controller.add_lvol_ha" in inspect.getsource(sr._create_landing_volume_unguarded) + assert "_create_landing_volume_unguarded(" in inspect.getsource(sr._create_landing_volume) assert probe < create, "the adopt/clear probe must run before the create" adopt = src.index("Adopting landing volume") assert probe < adopt < create diff --git a/simplyblock_core/test/test_replication_runner_resilience.py b/simplyblock_core/test/test_replication_runner_resilience.py index 87333f6db2..ebd4f4f7ad 100644 --- a/simplyblock_core/test/test_replication_runner_resilience.py +++ b/simplyblock_core/test/test_replication_runner_resilience.py @@ -144,6 +144,9 @@ def get_clusters(self): def get_job_tasks(self, cluster_id, reverse=False): return [_Task()] + def get_replication_tasks_for_snapshot(self, snapshot_id): + return [_Task()] + def get_task_by_id(self, uuid): raise KeyError(f"Task {uuid} not found") # vanished before the re-read From 3023534c20daa5ae12167aa195b87462de4cbd2c Mon Sep 17 00:00:00 2001 From: michael Date: Sat, 21 Nov 2026 22:15:54 +0100 Subject: [PATCH 9/9] chore(types): satisfy the type checker on the bf29/bf32 follow-ups Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/storage_node_ops.py | 7 +++-- .../test/test_migration_space_bf29.py | 5 ++-- .../test/test_repl_double_run_bf32.py | 28 +++++++++++++++---- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index b8c36bf152..907393bc35 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -12664,10 +12664,11 @@ def healthy_lvs_leadership_state(owner_id, lvs_name=None): except KeyError: return "skip", [], f"node {owner_id} unknown" if lvs_name and owner.lvstore != lvs_name: - owner = next((n for n in db.get_storage_nodes_by_cluster_id(owner.cluster_id) - if n.lvstore == lvs_name and not n.is_secondary_node), None) - if owner is None: + primary = next((n for n in db.get_storage_nodes_by_cluster_id(owner.cluster_id) + if n.lvstore == lvs_name and not n.is_secondary_node), None) + if primary is None: return "skip", [], f"no primary of {lvs_name} found" + owner = primary lvs_name = owner.lvstore if not lvs_name or owner.is_secondary_node or not owner.jm_vuid: return "skip", [], f"{owner.get_id()} holds no primary lvstore" diff --git a/simplyblock_core/test/test_migration_space_bf29.py b/simplyblock_core/test/test_migration_space_bf29.py index 37882d28b0..637ad7a6df 100644 --- a/simplyblock_core/test/test_migration_space_bf29.py +++ b/simplyblock_core/test/test_migration_space_bf29.py @@ -18,6 +18,7 @@ None of them touches client IO: only the migration's own snapshots wait. """ from types import SimpleNamespace +from typing import cast from unittest.mock import MagicMock, patch import pytest @@ -314,7 +315,7 @@ def test_bf29_a_held_round_is_not_measured_twice(monkeypatch, clock, real_rules) with pytest.raises(TaskDefer, match="waits for 1 deletion"): runner._shrink_step(task.frozen_view(), lvol) assert task.function_params["shrink_round_times"] == [20.0, 10.0] - runner._take_shrink_snapshot.assert_not_called() + cast(MagicMock, runner._take_shrink_snapshot).assert_not_called() def test_bf29_the_endgame_is_not_entered_while_its_first_round_would_wait( @@ -349,7 +350,7 @@ def test_bf29_no_round_snapshot_with_the_source_nearly_full(monkeypatch, clock, err, _ = runner._take_round(task.frozen_view(), _lvol("LV1", "LVS_1")) assert err assert len(events) == 1 - runner._take_shrink_snapshot.assert_not_called() + cast(MagicMock, runner._take_shrink_snapshot).assert_not_called() assert fake.rpc_calls == 0, "under the floor there is nothing to ask SPDK" diff --git a/simplyblock_core/test/test_repl_double_run_bf32.py b/simplyblock_core/test/test_repl_double_run_bf32.py index def4fae636..7c09b0a495 100644 --- a/simplyblock_core/test/test_repl_double_run_bf32.py +++ b/simplyblock_core/test/test_repl_double_run_bf32.py @@ -78,6 +78,22 @@ def _repl_task(uuid, snap=SNAP, status=JobSchedule.STATUS_NEW, date=1795237024, # -- (1) one task per snapshot: the atomic claim -------------------------------- + +def _recorder(ran): + """A task_runner stand-in that records which task ran.""" + def run(task): + ran.append(task.uuid) + return True + return run + + +def _snapshot_creating(seen): + """A create stand-in that records the in-flight registry while it runs.""" + def create(*_a): + seen.append(dict(sr._LANDING_CREATING)) + return ("id", None) + return create + def test_bf32_second_task_of_a_snapshot_is_refused_by_the_claim(dbc): first, created = dbc.ensure_snapshot_replication_task(_repl_task("5571400c")) assert (first, created) == ("5571400c", True) @@ -180,9 +196,9 @@ def test_bf32_only_one_of_two_live_tasks_of_a_snapshot_is_dispatched(status): the old RUNNING-only yield let both through.""" older = _repl_task("5571400c", status=status, date=100) newer = _repl_task("a6f9b7b7", status=status, date=101) - ran = [] + ran: list[str] = [] with patch.object(sr, "db", _TaskDB([older, newer])), \ - patch.object(sr, "task_runner", side_effect=lambda t: ran.append(t.uuid) or True): + patch.object(sr, "task_runner", side_effect=_recorder(ran)): assert sr._dispatch(newer) is True assert sr._dispatch(older) is True assert ran == ["5571400c"] @@ -191,9 +207,9 @@ def test_bf32_only_one_of_two_live_tasks_of_a_snapshot_is_dispatched(status): def test_bf32_the_task_owning_a_landing_volume_runs_first(): older = _repl_task("5571400c", date=100) newer = _repl_task("a6f9b7b7", date=101, remote_lvol_id="5eeb989b") - ran = [] + ran: list[str] = [] with patch.object(sr, "db", _TaskDB([older, newer])), \ - patch.object(sr, "task_runner", side_effect=lambda t: ran.append(t.uuid) or True): + patch.object(sr, "task_runner", side_effect=_recorder(ran)): sr._dispatch(older) sr._dispatch(newer) assert ran == ["a6f9b7b7"] @@ -315,9 +331,9 @@ def test_bf32_cleanup_does_not_delete_a_young_landing_volume(): def test_bf32_a_landing_volume_mid_create_is_registered_while_add_lvol_ha_runs(): - seen = [] + seen: list[dict] = [] with patch.object(sr, "_create_landing_volume_unguarded", - side_effect=lambda *a: seen.append(dict(sr._LANDING_CREATING)) or ("id", None)): + side_effect=_snapshot_creating(seen)): assert sr._create_landing_volume(REP_NAME, None, None, None) == ("id", None) assert seen == [{REP_NAME: 1}] assert REP_NAME not in sr._LANDING_CREATING