Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions simplyblock_core/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -711,6 +711,41 @@ def get_config_var(name, default=None):
# released, the volume replicates on its ordinary cadence, and the endgame is
# retried after this long (the abandon burns one attempt of the task's budget).
REPL_CUTOVER_ENDGAME_BACKOFF_SEC = 300
# --- a migration must not fill its source cluster (bed bf29, 2026-11-20) ---
# Five endgames of one source cluster at once, under sustained overwrite, took
# a shrink snapshot every 15-30 s each; every one pinned what the client
# overwrote after it, and their deletions fell behind: 41% -> 99% in 16 min.
# See simplyblock_core/migration_space.py. These hold back only the
# migration's own snapshots -- never client IO or client volume creation.
#
# Cutover endgames (shrink-round phase) running at once per SOURCE cluster.
# The others replicate on their ordinary cadence while they wait for a slot.
REPL_CUTOVER_MAX_ENDGAMES_PER_CLUSTER = 1
# An endgame takes its next round snapshot only once none of the volume's
# snapshots is still in deletion -- for at most this long per wait: a deletion
# that is stuck (no leader, a clone in deletion) must not stall the endgame
# into its shrink deadline on its own. The free-space check still applies.
REPL_ENDGAME_DELETION_WAIT_MAX_SEC = 300
# A migration snapshot needs this many times the bytes written to the volume
# since its last snapshot free on the source (the snapshot pins them, and the
# client overwrites about as much again while the round transfers) ...
REPL_MIGRATION_HEADROOM_FACTOR = 2
# ... and never less than this share of the cluster (other volumes' writes,
# and the capacity record being up to a collector interval old).
REPL_MIGRATION_MIN_FREE_PCT = 5
# A cluster capacity record older than this is no reading (the collector
# writes one every DEV_STAT_COLLECTOR_INTERVAL_SEC).
REPL_MIGRATION_CAPACITY_STALE_SEC = 120
# Without a usable capacity reading a migration snapshot waits this long, then
# goes ahead without the check (logged): a broken stats collector must not
# stop migrations for good.
REPL_MIGRATION_CAPACITY_UNKNOWN_WAIT_SEC = 120
# A cutover endgame whose target node is down before its clone exists cannot
# progress, and holds its volume out of ordinary replication while it waits
# (bed bf29 R3b: RPO 1280 s). After this long it is given up -- no attempt
# burned, no back-off -- and the volume replicates on its cadence until the
# target is back.
REPL_CUTOVER_TARGET_DOWN_GRACE_SEC = 60
# --- cutover source throttle ----------------------------------------------
# A round converges to overhead / (1 - W/T): with the application writing at
# W and the delta transferring at T, no number of rounds gets under the
Expand Down Expand Up @@ -896,6 +931,14 @@ def get_config_var(name, default=None):
# leader within about a minute of its primary going down (crashed or shut
# down); a wait beyond this means no member could be proven safe to lead.
REPL_TARGET_LEADERLESS_ALERT_SEC = 180
# A transfer that has waited this long for a target lvstore whose members are
# ALL online asks the control plane's guarded leaderless recovery
# (storage_node_ops.restore_healthy_lvs_leadership) to give it a leader, then
# at most every REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC. A primary SPDK that
# dropped the leadership itself takes it back only on IO, and a target gets
# none while replication waits (bf32 2026-11-21: LVS_7/LVS_10, 76 min).
REPL_TARGET_LEADERLESS_RECOVER_SEC = 60
REPL_TARGET_LEADERLESS_RECOVER_EVERY_SEC = 120
# Pass interval of the snapshot replication runner while any transfer or
# merge is in flight; TASK_EXEC_INTERVAL_SEC when idle. Not sub-second: each
# pass reads the task table (see test_transfer_completion_latency).
Expand All @@ -911,6 +954,13 @@ def get_config_var(name, default=None):
REPL_RUNNER_WORKERS = int(os.getenv("REPL_RUNNER_WORKERS", "8"))
# Pause before a task that returned a failure is dispatched again.
REPL_RUNNER_RETRY_PAUSE_SEC = 3.0
# An in_creation landing volume (REP_*) whose create is not running in this
# runner and that no other live task of its snapshot may be creating is taken
# for a dead attempt's leftover and deleted only after this long. Landing
# creates take up to ~50 s on a loaded target (bed bf13); bed bf32 (2026-11-21)
# deleted one 2 s into its create, which left a stale registration on the
# secondaries.
REPL_LANDING_ABANDON_SEC = int(os.getenv("REPL_LANDING_ABANDON_SEC", "600"))
# Requests a replication transfer keeps in flight. A full transfer sends one
# whole cluster per request (16 x 2 MiB = 32 MiB in flight); a delta
# (allow_partial) sends at most 64 KiB per request, so it needs a deeper window
Expand Down
23 changes: 23 additions & 0 deletions simplyblock_core/controllers/lvol_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,3 +199,26 @@ def replication_target_leaderless(lvol, lvs_name, waited_s, why, caused_by=ec.CA
f"{lvs_name} has had no leader for {int(waited_s)}s ({why})"),
node_id=lvol.get_id(),
event_level=EventObj.LEVEL_ERROR)


def migration_space_low(lvol, reason, caused_by=ec.CAUSED_BY_MONITOR):
"""A migration snapshot of *lvol* waits for free space on its source
cluster (simplyblock_core/migration_space.py). Warning level: client IO
is not affected, only the migration's own snapshots wait. Raised once per
wait, when it starts."""
from simplyblock_core.models.events import EventObj
try:
cluster_id = DBController().get_pool_by_id(lvol.pool_uuid).cluster_id
except Exception as e:
logger.error(f"No cluster could be resolved for lvol {lvol.get_id()}; event not logged: {e}")
return
ec.log_event_cluster(
cluster_id=cluster_id,
domain=ec.DOMAIN_CLUSTER,
event=ec.EVENT_REPLICATION_STALLED,
db_object=lvol,
caused_by=caused_by,
message=(f"Migration of LVol {lvol.get_id()} is waiting for free space on its "
f"source cluster: {reason}"),
node_id=lvol.get_id(),
event_level=EventObj.LEVEL_WARN)
23 changes: 20 additions & 3 deletions simplyblock_core/controllers/ns_host_guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,22 @@ def connected_hosts(nqn: str, node_ids: Iterable[str], db=None) -> set | None:

def stranded_hosts(nqn: str, removing: Iterable[str], keeping: Iterable[str],
db=None) -> set | None:
"""Hosts connected to *nqn* on the *removing* nodes but on none of the
*keeping* nodes; None when that could not be established."""
"""Hosts connected to *nqn* on the *removing* nodes that have no controller
on the *keeping* nodes; None when that could not be established.

A host NQN does not identify a machine: one Kubernetes node connects with
the kernel default (``/etc/nvme/hostnqn``) on one path and with the
simplyblock form derived from its node UID on another (bed bf28,
2026-11-19: every client reached the source as
``nqn.2014-08.io.simplyblock:uuid:<node uid>`` and the early-prepared
target as ``nqn.2014-08.org.nvmexpress:uuid:<machine id>``, with different
host ids), and nothing records which forms belong together. So hosts are
matched by name where they can be, and every unmatched host on the keeping
side is taken to be one unmatched host of the removing side under its
other name. Hosts are stranded only when the removing side has more
unmatched hosts than the keeping side can account for; all of the removing
side's unmatched hosts are then reported, since which of them is the
stranded one cannot be told."""
gone = connected_hosts(nqn, removing, db=db)
if gone is None:
return None
Expand All @@ -72,7 +86,10 @@ def stranded_hosts(nqn: str, removing: Iterable[str], keeping: Iterable[str],
stay = connected_hosts(nqn, keeping, db=db)
if stay is None:
return None
return gone - stay
unmatched_gone, unmatched_stay = gone - stay, stay - gone
if len(unmatched_gone) <= len(unmatched_stay):
return set()
return unmatched_gone


def removal_blocked(nqn: str, removing: Iterable[str], keeping: Iterable[str],
Expand Down
10 changes: 10 additions & 0 deletions simplyblock_core/controllers/tasks_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -421,6 +421,16 @@ def _add_task(function_name, cluster_id, node_id, device_id,
task_obj.function_params = function_params
task_obj.max_retry = max_retry
task_obj.status = JobSchedule.STATUS_NEW
if function_name == JobSchedule.FN_SNAPSHOT_REPLICATION:
# The scan above is a check-then-write: two replication workers
# enqueueing the same chain ancestor 100 ms apart both passed it and
# wrote two tasks for one snapshot (bed bf32 2026-11-21). The write
# itself is the atomic claim.
active_id, created = db.ensure_snapshot_replication_task(task_obj)
if not created:
logger.info(f"Task found, skip adding new task: {active_id}")
return False
return task_obj.uuid
task_obj.write_to_db(db.kv_store)
if send_to_cluster_log:
tasks_events.task_create(task_obj)
Expand Down
40 changes: 40 additions & 0 deletions simplyblock_core/db_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -923,6 +923,46 @@ def ensure_sync_resync_task(self, task: JobSchedule, lvs_name: str) -> tuple[str
return fdb.transactional(DBController._ensure_sync_resync_task_tx)(
self, self.kv_store, task, lvs_name, JobSchedule.active_indexes(self.kv_store))

# repl_snap_task/<cluster_id>/<snapshot_id> -> db key of the snapshot's
# replication task. Read and written in the transaction that creates the
# task, so two creators of the same snapshot's task conflict and the retry
# finds the first one's (bed bf32 2026-11-21).
REPL_SNAP_TASK_PREFIX = "repl_snap_task"

def _repl_snap_task_key(self, cluster_id, snapshot_id):
return f"{self.REPL_SNAP_TASK_PREFIX}/{cluster_id}/{snapshot_id}".encode()

def _ensure_snapshot_replication_task_tx(self, tr, task, index_list):
claim_key = self._repl_snap_task_key(
task.cluster_id, task.function_params["snapshot_id"])
raw = tr.get(claim_key).wait()
if raw is not None and raw.present():
current = BaseModel._read_record(tr, bytes(raw), JobSchedule)
if (current is not None and current.status != JobSchedule.STATUS_DONE
and not current.canceled):
return current.uuid, False
task.updated_at = str(datetime.datetime.now(datetime.UTC))
self._put_tx(tr, task, index_list)
tr[claim_key] = task.get_db_id().encode()
return task.uuid, True

def ensure_snapshot_replication_task(self, task: JobSchedule) -> tuple[str, bool]:
"""Create ``task`` (an FN_SNAPSHOT_REPLICATION task) unless its
snapshot already has one that is neither done nor canceled -- checked
and written in ONE transaction with the snapshot's claim key, so
concurrent callers never create two tasks for one snapshot.

bed bf32 (2026-11-21): two replication workers enqueued the same chain
ancestor 100 ms apart; each saw no task in its scan and each wrote
one. The two tasks then ran side by side: one created the landing
volume while the other deleted it as "half-created", leaving a stale
registration on the secondaries that later crashed SPDK.

Returns ``(uuid of the snapshot's active task, whether it is task)``.
"""
return self._transact(DBController._ensure_snapshot_replication_task_tx,
task, JobSchedule.active_indexes(self.kv_store))

def _finish_sync_resync_tx(self, tr, task_key, lvs_name, seq_limit, result, duration_sec,
task_indexes):
task = BaseModel._read_record(tr, task_key, JobSchedule)
Expand Down
Loading
Loading