From 526a4e7477b8d4b003656e90d4dfd1742d3cc6a3 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 30 Oct 2026 16:54:08 +0100 Subject: [PATCH 01/50] fix(lblk): uniform-only device growth, smallest factor across all LUNs Replaces the reweight/rebalance after uneven growth (8fd013c8c): the weight of an existing device in a live distrib map is never changed. - cluster grow-devices (CLI, v2 POST /clusters/{id}/grow-devices) rescans every data device and grows all of them by the SMALLEST growth factor of their block devices (1.5 / 2.0 -> every device grows by 1.5). A device that did not grow stops all growth; so does an offline device or node. Sizes are committed only after every alceml reopened, so a failure in between leaves the cluster uniform; repeated runs are no-ops. - Cluster-map weights (placement_size) never change. Removed: rebalance option, --no-rebalance, ?rebalance=, Cluster.reweight_pending, map push and rebalance queueing, devices_reweighted event. - Single-device growth (sn resize-device, v2 .../devices/{id}/resize) is removed: growth is cluster-wide only. - NVMeDevice.backend_size records the rescanned block-device size. alceml always spans the whole block device (no size limit: alceml takes blockcnt*blocklen of the backend on create and on resize reopen), so the capacity statistics scale npages_nmax down to the device size; the rest is reported as unused_size. With frozen weights placement fills every device in proportion to its weight, so the rest is never reached. - A block device smaller than its device is refused, logged as an error event, and raises the new device_shrunk alert until a rescan sees the restored size. Co-Authored-By: Claude Opus 5.5 --- simplyblock_cli/cli-reference.yaml | 33 +- simplyblock_cli/cli.py | 10 +- simplyblock_cli/clibase.py | 49 +- .../controllers/alerts_controller.py | 14 + .../controllers/device_controller.py | 368 ++++++-------- simplyblock_core/controllers/device_events.py | 22 +- simplyblock_core/models/cluster.py | 5 - simplyblock_core/models/nvme_device.py | 22 +- .../services/capacity_and_stats_collector.py | 7 + simplyblock_core/storage_node_ops.py | 7 +- simplyblock_web/api/v2/cluster/__init__.py | 37 +- .../api/v2/cluster/storage_node/device.py | 30 -- tests/integration/test_lblk_device_resize.py | 83 +-- tests/unit/test_lblk_device_resize.py | 473 ++++++++---------- .../unit/web/api/v2/test_device_endpoints.py | 75 ++- 15 files changed, 554 insertions(+), 681 deletions(-) diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 9505147527..29e88f6b97 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -566,19 +566,6 @@ commands: dest: force type: bool action: store_true - - name: resize-device - help: "Grows a Linux block device (lblk) online to the size of its block device." - usage: > - Re-reads the size of the device's block device on the storage node's host, for example, after its LUN was - grown on the storage array, and grows the storage device online without interrupting I/O. Shrinking is - refused. The placement weight of the device keeps its original size, so the added capacity is only fully - usable once all data devices of the cluster have grown by the same factor, or after cluster grow-devices - has updated the weights of unevenly grown devices and rebalanced the cluster. - arguments: - - name: "device_id" - help: "The storage device id." - dest: device_id - type: str - name: add-device help: "Adds a new storage device." usage: > @@ -1446,26 +1433,20 @@ commands: type: str completer: _completer_get_cluster_list - name: grow-devices - help: "Grows every Linux block device (lblk) of the cluster online to the size of its block device." + help: "Grows all Linux block devices (lblk) of the cluster online by the same factor." usage: > - Re-reads the size of every online data device of an lblk cluster and grows the devices whose block device - has grown, one at a time. Used after the LUNs of the cluster were grown on the storage array. When all - devices have grown by the same factor, nothing else changes. When they have grown unevenly, the device - weights are set to the new sizes and the cluster is rebalanced, which re-places a large part of the data - in the background; this needs an active cluster with all nodes and devices online and no running - migration. Grow all LUNs on the array first, then run this command once. + Re-reads the size of the block device of every data device of an lblk cluster, for example, after the + LUNs were grown on the storage array, and grows every device online by the same factor: the smallest + growth of any of the block devices. With three LUNs grown by 1.5 and three by 2.0, every device grows by + 1.5 and the rest stays unused. If one block device did not grow, nothing grows. Device weights and data + placement never change. All data devices and their nodes must be online. A block device smaller than its + device is refused and raises an alert. Grow all LUNs on the array first, then run this command once. arguments: - name: "cluster_id" help: "The cluster id." dest: cluster_id type: str completer: _completer_get_cluster_list - - name: "--no-rebalance" - help: "Keep the device weights after an uneven growth and do not rebalance the cluster." - dest: no_rebalance - type: bool - default: false - action: store_true - name: show help: "Shows a cluster's statistics." arguments: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index f7bc94312a..759634b9d3 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -60,7 +60,6 @@ def init_storage_node(self): self.init_storage_node__device_hang(subparser) self.init_storage_node__get_device(subparser) self.init_storage_node__restart_device(subparser) - self.init_storage_node__resize_device(subparser) self.init_storage_node__add_device(subparser) self.init_storage_node__remove_device(subparser) self.init_storage_node__set_failed_device(subparser) @@ -254,10 +253,6 @@ def init_storage_node__restart_device(self, subparser): subcommand.add_argument('device_id', help='The storage device id.', type=str) subcommand.add_argument('--force', help='Force restart.', dest='force', action='store_true') - def init_storage_node__resize_device(self, subparser): - subcommand = self.add_sub_command(subparser, 'resize-device', 'Grows a Linux block device (lblk) online to the size of its block device.') - subcommand.add_argument('device_id', help='The storage device id.', type=str) - def init_storage_node__add_device(self, subparser): subcommand = self.add_sub_command(subparser, 'add-device', 'Adds a new storage device.') subcommand.add_argument('device_id', help='The storage device id.', type=str) @@ -553,9 +548,8 @@ def init_cluster__complete_expand(self, subparser): subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list def init_cluster__grow_devices(self, subparser): - subcommand = self.add_sub_command(subparser, 'grow-devices', 'Grows every Linux block device (lblk) of the cluster online to the size of its block device.') + subcommand = self.add_sub_command(subparser, 'grow-devices', 'Grows all Linux block devices (lblk) of the cluster online by the same factor.') subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list - subcommand.add_argument('--no-rebalance', help='Keep the device weights after an uneven growth and do not rebalance the cluster.', default=False, dest='no_rebalance', action='store_true') def init_cluster__show(self, subparser): subcommand = self.add_sub_command(subparser, 'show', 'Shows a cluster\'s statistics.') @@ -1563,8 +1557,6 @@ def run(self): ret = self.storage_node__get_device(sub_command, args) elif sub_command in ['restart-device']: ret = self.storage_node__restart_device(sub_command, args) - elif sub_command in ['resize-device']: - ret = self.storage_node__resize_device(sub_command, args) elif sub_command in ['add-device']: ret = self.storage_node__add_device(sub_command, args) elif sub_command in ['remove-device']: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 2ff91c17ab..d83687e0db 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -448,20 +448,6 @@ def storage_node__reset_device(self, sub_command, args): def storage_node__restart_device(self, sub_command, args): return device_controller.restart_device(args.device_id, args.force) - def storage_node__resize_device(self, sub_command, args): - try: - result = device_controller.resize_device(args.device_id) - except ValueError as e: - print(e) - return False - if not result["resized"]: - return f"Device size unchanged: {utils.humanbytes(result['new_size'])}" - out = (f"Device grown from {utils.humanbytes(result['old_size'])} " - f"to {utils.humanbytes(result['new_size'])}") - if result["warning"]: - out += f"\nWarning: {result['warning']}" - return out - def storage_node__add_device(self, sub_command, args): return device_controller.add_device(args.device_id) @@ -767,25 +753,22 @@ def cluster__change_name(self, sub_command, args): return True def cluster__grow_devices(self, sub_command, args): - result = device_controller.grow_cluster_devices( - args.cluster_id, rebalance=not getattr(args, 'no_rebalance', False)) - lines = [f"{r['device_id']}: {utils.humanbytes(r['old_size'])} -> {utils.humanbytes(r['new_size'])}" - for r in result["devices"] if r["resized"]] - lines += [f"{dev_id}: failed: {err}" for dev_id, err in result["errors"].items()] - rebalance = result.get("rebalance") - if rebalance and rebalance["started"]: - lines.append("The devices have grown unevenly: device weights set to the new sizes, " - "cluster rebalancing started.") - elif rebalance and rebalance["weights_updated"]: - lines.append(f"The devices have grown unevenly: device weights set to the new sizes. " - f"{rebalance['reason']}") - elif rebalance: - lines.append(f"Warning: the devices have grown unevenly. {rebalance['reason']}") - elif not result["uniform"]: - lines.append("Warning: the data devices of the cluster have grown unevenly; the added capacity is " - "only fully usable once all of them have grown by the same factor, or after " - "grow-devices without --no-rebalance has updated the weights.") - return "\n".join(lines) or "No device has grown" + try: + result = device_controller.grow_cluster_devices(args.cluster_id) + except ValueError as e: + print(e) + return False + lines = [f"{dev_id}: {err}" for dev_id, err in result["errors"].items()] + if not result["grown"]: + lines.append(result["reason"]) + return "\n".join(lines) + lines.append(f"All data devices grown by factor {result['factor']:.4f}:") + for r in result["devices"]: + line = f"{r['device_id']}: {utils.humanbytes(r['old_size'])} -> {utils.humanbytes(r['new_size'])}" + if r["unused_size"]: + line += f" ({utils.humanbytes(r['unused_size'])} of the block device unused)" + lines.append(line) + return "\n".join(lines) def cluster__complete_expand(self, sub_command, args): cluster_ops.cluster_expand(args.cluster_id) diff --git a/simplyblock_core/controllers/alerts_controller.py b/simplyblock_core/controllers/alerts_controller.py index 0607d4f2f4..48edbdb79d 100644 --- a/simplyblock_core/controllers/alerts_controller.py +++ b/simplyblock_core/controllers/alerts_controller.py @@ -72,6 +72,7 @@ CLUSTER_PROV_CAPACITY_CRITICAL = "cluster_provisioned_capacity_critical" DEVICE_SLOW = "device_slow" DEVICE_UNHEALTHY = "device_unhealthy" +DEVICE_SHRUNK = "device_shrunk" SEVERITY_CRITICAL = "critical" SEVERITY_WARNING = "warning" @@ -290,6 +291,19 @@ def _device_alerts(cluster_id, nodes) -> list[dict]: label = _node_label(node) for dev in (node.nvme_devices or []): + # A block device (LUN) seen smaller than its storage device by + # the last rescan (device_controller.grow_cluster_devices). It is + # a state: it resolves when a rescan sees the restored size. + backend_size = getattr(dev, "backend_size", 0) or 0 + if 0 < backend_size < (getattr(dev, "size", 0) or 0): + alerts.append(_alert( + DEVICE_SHRUNK, SEVERITY_CRITICAL, + f"block device of device {dev.get_id()} on node {label} shrank to " + f"{utils.humanbytes(backend_size)}, below the device size of " + f"{utils.humanbytes(dev.size)}: restore the LUN size", + cluster_id, node_id=node.get_id(), device_id=dev.get_id(), + device_status=dev.status, + serial_number=getattr(dev, "serial_number", ""))) # Removed counts: a device that vanished from an otherwise # healthy node is a failure or a physical pull, and either way # somebody has to look. The one removal that is not is the diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index 1326cd3649..1ad963275b 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -7,7 +7,6 @@ from simplyblock_core.controllers import device_events, tasks_controller from simplyblock_core.db_controller import DBController from simplyblock_core.models.cluster import Cluster -from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.nvme_device import JMDevice, NVMeDevice from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.prom_client import PromClient @@ -1912,35 +1911,34 @@ def _apply(dev): # --------------------------------------------------------------------------- -# Online growth of lblk devices (a LUN grown on its storage array). +# Online growth of lblk devices (LUNs grown on their storage array). # # The base bdev of an lblk device is an SPDK AIO bdev over a kernel block # device, and alceml sits directly on it. alceml can swap its backing bdev # for a larger one without losing state: resize_begin holds new I/O, drains # pending I/O and closes the backing bdev; resize_end reopens it and takes -# its new size, so the pages beyond the old end become allocatable. Layers -# above alceml see a fixed virtual size and need no change. +# the size of the whole block device (alceml has no size limit of its own), +# so the pages beyond the old end become allocatable. Layers above alceml see +# a fixed virtual size and need no change. # -# Growing a device does not change its cluster-map weight. The weight is -# derived from NVMeDevice.placement_size, the size the device had when it was -# placed, and distrib's map spec (message_format_rpcs__distrib__v5.txt, -# weight_int) asks for weights never to change, because a change re-places -# 50% or more of the data. Weights only matter relative to each other, so when -# every data device grows by the same factor (the normal case: equal LUNs grown -# together on the array), nothing has to change and nothing moves. +# Growth is uniform only. The cluster-map weight of a device is derived from +# NVMeDevice.placement_size, the size the device had when it was placed, and +# never changes. Weights only matter relative to each other, so the cluster +# can grow without any data movement exactly when every data device grows by +# the same factor. grow_cluster_devices therefore rescans every data device, +# takes the SMALLEST growth factor across all of them, and grows every device +# by exactly that factor: with three LUNs grown by 1.5 and three by 2.0, +# every device grows by 1.5. A device whose block device did not grow keeps +# the factor at 1.0, and nothing grows. The rest of a larger LUN stays unused: +# placement fills every device in proportion to its weight, so it is never +# reached, and the capacity statistics leave it out (NVMeDevice.backend_size). # -# After an uneven growth the frozen weights no longer match the capacities, -# and the added capacity is not fully usable. grow_cluster_devices then sets -# every weight to the device's current size, pushes the new map to all nodes, -# and starts the cluster rebalance (rebalance_after_growth). That re-places a -# large part of the data, in the background and within the migration share of -# the QoS budget. Correctness does not depend on the static placement: distrib -# finds data by its recorded location, including locations that were correct -# under an earlier map (alg_remapped_ranges.hpp, get_location_latest, "STALE"), -# which is the same mechanism a device addition relies on. +# A block device smaller than its storage device is never accepted: the +# rescan records it, logs an error event, raises the device_shrunk alert, and +# nothing grows until the LUN is restored and rescanned. # -# A device that enters the cluster later gets a weight on the same scale as the -# grown devices (storage_node_ops.scaled_placement_size), so a uniform growth +# A device that enters the cluster later gets a weight on the same scale as +# the grown devices (storage_node_ops.scaled_placement_size), so a growth # followed by an expansion does not overfill the new device. # --------------------------------------------------------------------------- @@ -1949,6 +1947,10 @@ def _apply(dev): RESIZE_RPC_TIMEOUT_SEC = 120 #: Attempts to reopen the backing bdev before the device is given up. RESIZE_END_ATTEMPTS = 3 +#: A cluster-wide growth factor below this counts as no growth. +GROWTH_MIN_FACTOR = 1.001 +#: Grown device sizes are rounded down to this alignment. +GROWTH_SIZE_ALIGN = 1024 * 1024 def cluster_growth_uniformity(cluster_id): @@ -1958,9 +1960,25 @@ def cluster_growth_uniformity(cluster_id): return storage_node_ops.cluster_growth_ratios(DBController(), cluster_id) -def _check_resizable(db_controller, dev, snode): +def _growth_devices(db_controller, cluster_id): + """``(node, device)`` for every data device of the cluster that counts + for the growth factor: the same set as the cluster-map weights.""" + pairs = [] + for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): + if node.is_secondary_node: + continue + for dev in node.nvme_devices: + if dev.status in (NVMeDevice.STATUS_JM, NVMeDevice.STATUS_NEW, + NVMeDevice.STATUS_FAILED, NVMeDevice.STATUS_FAILED_AND_MIGRATED): + continue + if dev.cluster_device_order < 0: + continue + pairs.append((node, dev)) + return pairs + + +def _check_resizable(cluster, dev, snode): """Raise ValueError when ``dev`` cannot be grown online right now.""" - cluster = db_controller.get_cluster_by_id(snode.cluster_id) if cluster.device_mode != constants.DEVICE_MODE_LBLK or dev.bdev_type != "aio": raise ValueError("Online resizing is only supported for Linux block devices (lblk device mode)") if dev.status != NVMeDevice.STATUS_ONLINE: @@ -1977,6 +1995,18 @@ def _check_resizable(db_controller, dev, snode): raise ValueError("A restart task is active for the device") +def _rescan_size(snode, dev): + """Make the storage node's host re-read the size of the device's block + device (SCSI or NVMe rescan) and return it in bytes.""" + path = dev.by_id_path or dev.device_path + if not path: + raise ValueError("No block device path known for the device") + result, err = snode.client(timeout=60, retry=1).rescan_block_device(path) + if err or not result: + raise ValueError(f"Rescan of {path} failed: {err}") + return int(result["size"]) + + def _reopen_after_resize(rpc_client, dev): """Leave alceml's resize state with the backing bdev reopened. Returns True on success. On persistent failure alceml is told to fail its I/O, @@ -2009,43 +2039,10 @@ def _resize_failed(device_id, dev, msg, set_unavailable): return ValueError(msg) -def resize_device(device_id): - """Grow an lblk device online to the size its block device now has. - - The kernel is made to re-read the size first (SCSI or NVMe rescan on the - storage node's host). A smaller size is refused. Returns a dict with - ``device_id``, ``old_size``, ``new_size``, ``placement_size``, - ``resized``, and the cluster's ``uniform`` growth flag with a - ``warning`` when it is not uniform. Raises ValueError on refusal or - failure; the message says whether the device is unchanged.""" - db_controller = DBController() - try: - dev = db_controller.get_storage_device_by_id(device_id) - snode = db_controller.get_storage_node_by_id(dev.node_id) - except KeyError as e: - raise ValueError(f"Device not found: {device_id}") from e - _check_resizable(db_controller, dev, snode) - - path = dev.by_id_path or dev.device_path - if not path: - raise ValueError("No block device path known for the device") - result, err = snode.client(timeout=60, retry=1).rescan_block_device(path) - if err or not result: - raise ValueError(f"Rescan of {path} failed: {err}") - new_size = int(result["size"]) - old_size = dev.size - - if new_size < old_size: - raise ValueError( - f"The block device shrank from {old_size} to {new_size} bytes. Shrinking is not " - f"supported and destroys data: restore the LUN to at least {old_size} bytes") - if new_size == old_size: - uniform, _, _ = cluster_growth_uniformity(snode.cluster_id) - return {"device_id": device_id, "old_size": old_size, "new_size": new_size, - "placement_size": dev.weight_size(), "resized": False, "uniform": uniform, - "warning": ""} - - logger.info(f"Growing device {device_id} from {old_size} to {new_size} bytes") +def _reopen_alceml(snode, dev): + """Let alceml reopen its AIO bdev at the size the block device has now: + resize_begin -> bdev_aio_rescan -> resize_end. Raises ValueError.""" + device_id = dev.get_id() rpc_client = snode.rpc_client() try: rpc_client.alceml_bdev_backend_resize_begin( @@ -2066,151 +2063,112 @@ def resize_device(device_id): if rescan_error is not None: raise _resize_failed(device_id, dev, f"AIO rescan failed: {rescan_error}", False) - def _apply(d, n=new_size): - if d.placement_size <= 0: - d.placement_size = d.size - d.size = n - _atomic_device_set(db_controller, snode.get_id(), device_id, _apply) - fresh = db_controller.get_storage_device_by_id(device_id) - device_events.device_resized(fresh, old_size, new_size) - - uniform, lo, hi = cluster_growth_uniformity(snode.cluster_id) - warning = "" - if not uniform: - warning = ( - f"The data devices of the cluster have grown unevenly (growth factors {lo:.2f} to {hi:.2f}). " - "Placement keeps filling every device in proportion to its original size, so the added " - "capacity is only fully usable once all data devices have grown by the same factor, or " - "after cluster grow-devices has updated the weights and rebalanced the cluster.") - logger.warning(warning) - return {"device_id": device_id, "old_size": old_size, "new_size": new_size, - "placement_size": fresh.weight_size(), "resized": True, "uniform": uniform, - "warning": warning} - - -#: Tasks that move data across the cluster map or change it. New weights are -#: never pushed while one of them is open. -_PLACEMENT_TASKS = ( - JobSchedule.FN_DEV_MIG, JobSchedule.FN_NEW_DEV_MIG, JobSchedule.FN_FAILED_DEV_MIG, - JobSchedule.FN_BALANCING_AFTER_NODE_RESTART, JobSchedule.FN_BALANCING_AFTER_DEV_REMOVE, - JobSchedule.FN_BALANCING_AFTER_DEV_EXPANSION, JobSchedule.FN_CLUSTER_EXPAND, - JobSchedule.FN_LVOL_MIG, JobSchedule.FN_LVOL_BATCH_MIG, -) +def _growth_result(grown, factor, devices, errors, reason): + return {"grown": grown, "factor": round(factor, 6), "devices": devices, + "errors": errors, "reason": reason} -def _reweight_blocker(db_controller, cluster_id): - """Why new weights cannot be pushed now, or "" when they can. The new map - must reach every node before any data moves: a migration that runs while - nodes disagree about placement can lose writes.""" - cluster = db_controller.get_cluster_by_id(cluster_id) - if cluster.status != Cluster.STATUS_ACTIVE: - return f"the cluster is {cluster.status}, not active" - for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): - if node.status == StorageNode.STATUS_REMOVED: - continue - if node.status != StorageNode.STATUS_ONLINE: - return f"storage node {node.get_id()} is {node.status}" - for dev in node.nvme_devices: - if dev.status in (NVMeDevice.STATUS_JM, NVMeDevice.STATUS_NEW, - NVMeDevice.STATUS_FAILED, NVMeDevice.STATUS_FAILED_AND_MIGRATED): - continue - if dev.status != NVMeDevice.STATUS_ONLINE: - return f"device {dev.get_id()} is {dev.status}" - for task in db_controller.get_job_tasks(cluster_id): - if (task.function_name in _PLACEMENT_TASKS and task.status != JobSchedule.STATUS_DONE - and not task.canceled): - return f"task {task.uuid} ({task.function_name}) is still open" - return "" - - -def rebalance_after_growth(cluster_id): - """Set the cluster-map weight of every online data device to its current - size, push the new map to every node, and start the cluster rebalance. - - Refused (nothing changed) unless the cluster is active, every node and - data device is online, and no migration or expansion is open. If a map - push fails, the new weights stay recorded with ``reweight_pending`` and - the rebalance is not started; the next call pushes again. Returns - ``{"weights_updated": bool, "started": bool, "reason": str}``: - ``weights_updated`` once every node runs on the new weights, ``started`` - once the rebalance is queued, ``reason`` when either is missing.""" + +def _device_report(dev, old_size): + return {"device_id": dev.get_id(), "old_size": old_size, "new_size": dev.size, + "backend_size": dev.backend_size, "unused_size": dev.unused_size(), + "placement_size": dev.weight_size()} + + +def grow_cluster_devices(cluster_id): + """Grow every data device of an lblk cluster by the same factor: the + smallest growth of any of their block devices (see the section comment). + + Every data device must be online and growable; otherwise nothing grows. + All block devices are rescanned first and their sizes recorded. If one + of them is smaller than its device, the shrink is reported (event and + device_shrunk alert) and nothing grows. Otherwise every alceml reopens + its grown block device, and only when all of them succeeded is every + device size set to ``size * factor``; a failure in between leaves all + recorded sizes unchanged, so the cluster stays uniform and a re-run + finishes the job. Repeated runs without new growth change nothing. + + Returns ``{"grown": bool, "factor": float, "devices": [...], + "errors": {device_id: message}, "reason": str}``; every device entry has + ``device_id``, ``old_size``, ``new_size``, ``backend_size``, + ``unused_size`` and ``placement_size``.""" db_controller = DBController() cluster = db_controller.get_cluster_by_id(cluster_id) - blocker = _reweight_blocker(db_controller, cluster_id) - if blocker: - if cluster.reweight_pending: - return _rebalance_outcome(False, False, - f"New weights recorded, cluster map not pushed yet: {blocker}") - return _rebalance_outcome(False, False, f"Weights not updated: {blocker}") - - if not cluster.reweight_pending: - db_controller.atomic_update(cluster, lambda c: setattr(c, "reweight_pending", True)) - - def _apply(n): - for d in n.nvme_devices: - if d.status == NVMeDevice.STATUS_ONLINE: - d.placement_size = d.size - for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): - if node.status != StorageNode.STATUS_REMOVED: - db_controller.atomic_update(node, _apply) - logger.info(f"Cluster {cluster_id}: device weights set to the current device sizes") - - nodes = [n for n in db_controller.get_storage_nodes_by_cluster_id(cluster_id) - if n.status == StorageNode.STATUS_ONLINE] - failed = [n.get_id() for n in nodes if not distr_controller.send_cluster_map_to_node(n)] - if failed: - return _rebalance_outcome( - False, False, f"New weights recorded, but the cluster map push failed on {', '.join(failed)}; " - "run cluster grow-devices again to retry") - - db_controller.atomic_update(db_controller.get_cluster_by_id(cluster_id), - lambda c: setattr(c, "reweight_pending", False)) - # The same rebalance as POST /clusters/{id}/rebalance: one expansion - # migration per distrib, under a balancing master task. - queued = bool(nodes) and tasks_controller.add_device_mig_task_for_node(nodes[0].get_id()) - device_events.devices_reweighted(cluster_id, queued) - if not queued: - return _rebalance_outcome( - True, False, "Weights updated, but no rebalance was queued (no volume store holds data, or a " - "node removal is recorded); start it with the cluster rebalance API when needed") - return _rebalance_outcome(True, True, "") - - -def _rebalance_outcome(weights_updated, started, reason): - return {"weights_updated": weights_updated, "started": started, "reason": reason} - - -def grow_cluster_devices(cluster_id, rebalance=True): - """Grow every online lblk data device of a cluster to the size of its - block device, one device at a time. Devices whose size did not change - are left alone; a device that fails does not stop the others. - - When the devices have grown unevenly and ``rebalance`` is set, the - weights are updated and the cluster is rebalanced (rebalance_after_growth), - unless a device failed to grow. Returns ``{"devices": [...], "errors": - {...}, "uniform": bool, "rebalance": {...} | None}`` (see - rebalance_after_growth for the rebalance dict); - ``uniform`` describes the growth before any weight change.""" - db_controller = DBController() - results, errors = [], {} - for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): - if node.is_secondary_node: - continue - for dev in node.nvme_devices: - if dev.status != NVMeDevice.STATUS_ONLINE or dev.bdev_type != "aio": - continue - try: - results.append(resize_device(dev.get_id())) - except ValueError as e: - errors[dev.get_id()] = str(e) - uniform, _, _ = cluster_growth_uniformity(cluster_id) - outcome = None - pending = db_controller.get_cluster_by_id(cluster_id).reweight_pending - if rebalance and (not uniform or pending): - if errors and not pending: - outcome = _rebalance_outcome( - False, False, "Weights not updated: some devices failed to grow; fix them and run " - "cluster grow-devices again") - else: - outcome = rebalance_after_growth(cluster_id) - return {"devices": results, "errors": errors, "uniform": uniform, "rebalance": outcome} + if cluster.device_mode != constants.DEVICE_MODE_LBLK: + raise ValueError("Online resizing is only supported for Linux block devices (lblk device mode)") + pairs = _growth_devices(db_controller, cluster_id) + if not pairs: + return _growth_result(False, 1.0, [], {}, "The cluster has no data devices") + + errors = {} + for node, dev in pairs: + try: + _check_resizable(cluster, dev, node) + except ValueError as e: + errors[dev.get_id()] = str(e) + if errors: + return _growth_result(False, 1.0, [], errors, + "Not every data device can be grown; growth is uniform across all " + "data devices, so nothing was grown") + + backend = {} + for node, dev in pairs: + try: + backend[dev.get_id()] = _rescan_size(node, dev) + except ValueError as e: + errors[dev.get_id()] = str(e) + if errors: + return _growth_result(False, 1.0, [], errors, + "Not every block device could be rescanned; nothing was grown") + + for node, dev in pairs: + size = backend[dev.get_id()] + if dev.backend_size != size: + _atomic_device_set(db_controller, node.get_id(), dev.get_id(), + lambda d, s=size: setattr(d, "backend_size", s)) + dev.backend_size = size + shrunk = [(node, dev) for node, dev in pairs if dev.is_shrunk()] + if shrunk: + for _, dev in shrunk: + device_events.device_shrink_detected(dev, dev.backend_size) + errors[dev.get_id()] = ( + f"The block device shrank to {dev.backend_size} bytes, below the device size of " + f"{dev.size} bytes. Shrinking is not supported: restore the LUN to at least the " + f"device size") + return _growth_result(False, 1.0, [], errors, "A block device is smaller than its device; nothing was grown") + + factor = min(backend[dev.get_id()] / dev.size for _, dev in pairs) + if factor < GROWTH_MIN_FACTOR: + return _growth_result(False, 1.0, [_device_report(dev, dev.size) for _, dev in pairs], {}, + "Not every block device has grown; growth is uniform across all data " + "devices, so nothing was grown") + + targets = {} + for _, dev in pairs: + target = int(dev.size * factor) // GROWTH_SIZE_ALIGN * GROWTH_SIZE_ALIGN + targets[dev.get_id()] = min(max(target, dev.size), backend[dev.get_id()]) + + logger.info(f"Growing {len(pairs)} data devices of cluster {cluster_id} by factor {factor:.4f}") + for node, dev in pairs: + try: + _reopen_alceml(node, dev) + except ValueError as e: + errors[dev.get_id()] = str(e) + return _growth_result(False, factor, [], errors, + "A device failed to grow; all device sizes are unchanged, run " + "grow-devices again once the device is back") + + reports = [] + for node, dev in pairs: + old_size = dev.size + new_size = targets[dev.get_id()] + + def _apply(d, n=new_size): + if d.placement_size <= 0: + d.placement_size = d.size + d.size = n + _atomic_device_set(db_controller, node.get_id(), dev.get_id(), _apply) + fresh = db_controller.get_storage_device_by_id(dev.get_id()) + device_events.device_resized(fresh, old_size, new_size) + reports.append(_device_report(fresh, old_size)) + return _growth_result(True, factor, reports, {}, "") diff --git a/simplyblock_core/controllers/device_events.py b/simplyblock_core/controllers/device_events.py index c2278ce666..2e278feda2 100644 --- a/simplyblock_core/controllers/device_events.py +++ b/simplyblock_core/controllers/device_events.py @@ -106,18 +106,24 @@ def device_resized(device, old_size, new_size, caused_by=ec.CAUSED_BY_CLI): caused_by, ec.EVENT_STATUS_CHANGE) -def devices_reweighted(cluster_id, rebalancing, caused_by=ec.CAUSED_BY_CLI): - """Cluster-log entry for new device weights after an uneven growth of - lblk devices; the rebalance that follows is logged by its tasks.""" +def device_shrink_detected(device, backend_size, caused_by=ec.CAUSED_BY_CLI): + """ERROR-level cluster-log entry for a block device found smaller than + its storage device. The device_shrunk alert stands until a rescan sees + the block device at its full size again.""" + db_controller = DBController() + snode = db_controller.get_storage_node_by_id(device.node_id) ec.log_event_cluster( - cluster_id=cluster_id, + cluster_id=snode.cluster_id, domain=ec.DOMAIN_CLUSTER, event=ec.EVENT_STATUS_CHANGE, - db_object=DBController().get_cluster_by_id(cluster_id), + db_object=device, caused_by=caused_by, - message="Device weights set to the current device sizes after uneven growth" - + ("; rebalancing started" if rebalancing else ""), - node_id=cluster_id) + message=(f"Block device of the device shrank to {utils.humanbytes(backend_size)}, " + f"below the device size of {utils.humanbytes(device.size)}. Shrinking is not " + f"supported: restore the LUN to at least the device size"), + node_id=device.get_id(), + event_level=EventObj.LEVEL_ERROR, + storage_id=device.cluster_device_order) def device_resize_failed(device, message, caused_by=ec.CAUSED_BY_CLI): diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index b84f1293f9..74ad1798a1 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -338,11 +338,6 @@ def is_topology_owned(self) -> bool: # ``simplyblock_core.controllers.cluster_expansion.planner``. See the feature plan # ``single_node_expansion_plan.md`` for the schema. expand_state: dict = default_factory(dict) - # Set while new device weights (placement sizes after an uneven growth of - # lblk devices) are written but not yet pushed to every node and handed to - # a rebalance. A re-run of `cluster grow-devices` finishes the job. See - # device_controller.rebalance_after_growth. - reweight_pending: bool = False # State owned by release-specific upgrade plug-ins (see # simplyblock_core/release_upgrades/). Keys are plugin STATE_KEYs; a key # is set by the plugin's pre_update step (first thing `cluster update` diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index 0da8f8df24..54e815cb23 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -77,9 +77,17 @@ class NVMeDevice(BaseModel): # distrib placement weight of the device is derived from it, and distrib # requires a weight never to change once set (a change re-places 50% or # more of the data). ``size`` follows an online growth of an lblk device - # (device_controller.resize_device); this field keeps the weight frozen. - # 0 means the device was never resized, so ``size`` is still the weight. + # (device_controller.grow_cluster_devices); this field keeps the weight + # frozen. 0 means the device was never grown, so ``size`` is still the + # weight. placement_size: int = 0 + # Size in bytes of the device's block device (the LUN) when the cluster + # last rescanned it (device_controller.grow_cluster_devices); 0 = never + # rescanned. alceml always uses the whole block device, but all devices + # grow by the same factor only, so ``size`` may stay below this value; the + # rest is unused and left out of the capacity. A value below ``size`` means + # the block device shrank, which raises an alert. + backend_size: int = 0 # NVMe per-block metadata size in bytes, as reported by the bound SPDK bdev. # >=8 means alceml can run in cv_md_method (no read/write amplification). # 0 means alceml must use cv_fallback_method (extra md page per 2 MiB extent). @@ -118,6 +126,16 @@ def weight_size(self) -> int: size it had when it was placed, unchanged by later growth.""" return self.placement_size if self.placement_size > 0 else self.size + def unused_size(self) -> int: + """Bytes of the block device beyond ``size``: grown on the storage + array, but not usable because the other data devices of the cluster + grew by a smaller factor.""" + return max(0, self.backend_size - self.size) if self.size > 0 else 0 + + def is_shrunk(self) -> bool: + """The block device was last seen smaller than the device.""" + return 0 < self.backend_size < self.size + def __change_dev_connection_to(self, connecting_from_node): # Targeted single-record write. The previous implementation scanned # the WHOLE node table and full-object-wrote the owning node record diff --git a/simplyblock_core/services/capacity_and_stats_collector.py b/simplyblock_core/services/capacity_and_stats_collector.py index c19d240059..bb1fbc5e58 100644 --- a/simplyblock_core/services/capacity_and_stats_collector.py +++ b/simplyblock_core/services/capacity_and_stats_collector.py @@ -158,6 +158,13 @@ def add_device_stats(cl, device, capacity_dict, stats_dict, tick_rate=0): # size_prov and with the lvol/snapshot figures, which are effective by # construction. The raw numbers are kept in the *_raw fields. size_total_raw = int(capacity_dict['npages_nmax']*capacity_dict['pba_page_size']) + # alceml spans its whole block device (npages_nmax follows the backend + # size). After a LUN grew by more than the cluster-wide growth factor, + # only ``size`` of it is usable: placement fills every device in + # proportion to its frozen weight, so the rest is never reached. + backend_size, dev_size = getattr(device, "backend_size", 0), getattr(device, "size", 0) + if isinstance(backend_size, int) and isinstance(dev_size, int) and backend_size > dev_size > 0: + size_total_raw = size_total_raw * dev_size // backend_size size_used_raw = int(capacity_dict['npages_used']*capacity_dict['pba_page_size']) size_free_raw = size_total_raw - size_used_raw diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 6e3c20bec5..c276cbd72a 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -1262,7 +1262,8 @@ def cluster_growth_ratios(db_controller, cluster_id): """Growth of the data devices relative to their placement weight, as ``(uniform, min_ratio, max_ratio)``. Journal, new, and failed devices are ignored. A ratio above 1 means the device grew after it was placed (see - device_controller.resize_device).""" + device_controller.grow_cluster_devices, which grows all of them by the + same factor).""" ratios = [] for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): if node.is_secondary_node: @@ -1288,8 +1289,8 @@ def scaled_placement_size(db_controller, cluster_id, size): a common factor f, their weights still stand for their original sizes, so a new device is weighted as size / f; weighting it by its full size would give it f times its share of the data. Returns 0 (weight = size) when the - cluster has not grown, or has grown unevenly (then the next - grow-devices run sets every weight to its size anyway).""" + cluster has not grown. Growth is uniform by construction; should the + ratios ever disagree, the device keeps its full size as weight.""" uniform, lo, hi = cluster_growth_ratios(db_controller, cluster_id) factor = (lo + hi) / 2 if not uniform or factor <= 1.0 + GROWTH_UNIFORM_TOLERANCE or not size: diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index a4e16ab950..7a134e729e 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -332,29 +332,38 @@ def rebalance_cluster( cluster: Cluster) -> Response: return Response(status_code=204) -class DeviceRebalanceDTO(BaseModel): - weights_updated: bool - started: bool - reason: str +class DeviceGrowthEntryDTO(BaseModel): + device_id: str + old_size: int + new_size: int + backend_size: int + unused_size: int + placement_size: int class DeviceGrowthDTO(BaseModel): - devices: builtins.list[dict] + grown: bool + factor: float + devices: builtins.list[DeviceGrowthEntryDTO] errors: dict[str, str] - uniform: bool - rebalance: DeviceRebalanceDTO | None = None + reason: str @instance_api.post('/grow-devices', name='clusters:grow_devices', response_model=DeviceGrowthDTO) -def grow_devices(cluster: Cluster, rebalance: bool = True) -> DeviceGrowthDTO: - """Grow every online lblk data device of the cluster to the size of its - block device, one device at a time. Used after the LUNs of the cluster - were grown on the storage array. If they have grown unevenly, the device - weights are set to the new sizes and the cluster is rebalanced, unless - ``rebalance`` is false.""" +def grow_devices(cluster: Cluster) -> DeviceGrowthDTO: + """Grow all lblk data devices of the cluster online by the same factor. + + Every block device is rescanned (for example, after its LUN was grown on + the storage array), and every device grows by the smallest growth factor + of all of them; the rest of a larger block device stays unused + (`unused_size`). If one block device did not grow, nothing grows + (`grown` false, `reason`). Device weights and data placement never change. + A block device smaller than its device is refused, reported in `errors`, + and raises the device_shrunk alert. + """ if cluster.device_mode != constants.DEVICE_MODE_LBLK: raise HTTPException(409, 'Online device growth needs a cluster in lblk device mode') - return DeviceGrowthDTO(**device_controller.grow_cluster_devices(cluster.get_id(), rebalance=rebalance)) + return DeviceGrowthDTO(**device_controller.grow_cluster_devices(cluster.get_id())) instance_api.include_router(alert_api, prefix='/alerts') diff --git a/simplyblock_web/api/v2/cluster/storage_node/device.py b/simplyblock_web/api/v2/cluster/storage_node/device.py index c833155afb..3adfed29c3 100644 --- a/simplyblock_web/api/v2/cluster/storage_node/device.py +++ b/simplyblock_web/api/v2/cluster/storage_node/device.py @@ -4,7 +4,6 @@ from uuid import UUID from fastapi import APIRouter, HTTPException, Request, Response -from pydantic import BaseModel from sse_starlette import EventSourceResponse from simplyblock_core.controllers import device_controller @@ -134,35 +133,6 @@ def replace( ) -class DeviceResizeDTO(BaseModel): - device_id: str - old_size: int - new_size: int - placement_size: int - resized: bool - uniform: bool - warning: str = "" - - -@instance_api.post('/resize', name='clusters:storage_nodes:devices:resize', - response_model=DeviceResizeDTO, - responses={409: {"description": "The device cannot be grown"}}) -def resize(cluster: Cluster, storage_node: StorageNode, device: Device) -> DeviceResizeDTO: - """Grow an lblk device online to the size its block device now has. - - The storage node's host re-reads the size of the block device (for - example, a LUN grown on its storage array) and the device takes it over - without interrupting I/O. Shrinking is refused. The placement weight of - the device stays at its original size, so the added capacity is only - fully usable once all data devices of the cluster have grown by the same - factor; `uniform` and `warning` report it. - """ - try: - return DeviceResizeDTO(**device_controller.resize_device(device.get_id())) - except ValueError as e: - raise HTTPException(409, str(e)) from e - - @instance_api.post('/restart', name='clusters:storage_nodes:devices:restart', status_code=204, responses={204: {"content": None}}) def restart(cluster: Cluster, storage_node: StorageNode, device: Device, force: bool = False) -> Response: if not device_controller.restart_device(device.get_id(), force): diff --git a/tests/integration/test_lblk_device_resize.py b/tests/integration/test_lblk_device_resize.py index 18cf41b5c1..4dc0bce163 100644 --- a/tests/integration/test_lblk_device_resize.py +++ b/tests/integration/test_lblk_device_resize.py @@ -3,19 +3,20 @@ What runs REAL here: the FDB persistence of the grown device (size and the frozen placement_size), the cluster-map build from the persisted records, -and device_controller.resize_device / grow_cluster_devices end to end. What +and device_controller.grow_cluster_devices end to end. What is faked: SPDK RPC, the node agent (rescan_block_device), and the event fan-out. Scenarios: - 1. A grown device persists its new size and keeps its placement size, and - the cluster map built from the stored records carries the old weight. - 2. A second growth keeps the original placement size. - 3. grow_cluster_devices grows every data device by the same factor and - reports the cluster as uniform; the journal device is left alone. + 1. A uniform growth persists the new sizes and keeps the placement sizes, + and the cluster map built from the stored records carries the old + weights; a second growth keeps the original placement size. + 2. Mixed growth (1.5 / 2.0) grows every data device by 1.5, records the + unused rest, and leaves the journal device alone. + 3. A shrunk block device is persisted and nothing grows. 4. A failed reopen leaves the stored size unchanged and the device unavailable. - 5. placement_size survives an FDB round-trip and defaults to 0. + 5. placement_size and backend_size survive an FDB round-trip. """ import uuid as uuid_mod @@ -106,21 +107,22 @@ def test_placement_size_round_trip(self, db): _seed_node(db, [dev]) read = db.get_storage_device_by_id(dev.get_id()) assert read.placement_size == 0 + assert read.backend_size == 0 assert read.weight_size() == 100 * _GIB - def test_grown_device_keeps_cluster_map_weight(self, db): + def test_uniform_growth_keeps_cluster_map_weight(self, db): _seed_cluster(db) a, b = _aio_device("S1", 0), _aio_device("S2", 1) node = _seed_node(db, [a, b]) - agent = _agent({a.by_id_path: 300 * _GIB}) + agent = _agent({a.by_id_path: 300 * _GIB, b.by_id_path: 300 * _GIB}) - res = _run(agent, MagicMock(), device_controller.resize_device, a.get_id()) + res = _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) - assert res["resized"] is True - assert res["uniform"] is False + assert res["grown"] is True read = db.get_storage_device_by_id(a.get_id()) assert read.size == 300 * _GIB assert read.placement_size == 100 * _GIB + assert read.backend_size == 300 * _GIB fresh = db.get_storage_node_by_id(node.get_id()) cl_map = distr_controller.get_distr_cluster_map([fresh], fresh, "distr-1") @@ -133,9 +135,9 @@ def test_second_growth_keeps_original_placement(self, db): _seed_node(db, [a]) sizes = {a.by_id_path: 200 * _GIB} agent = _agent(sizes) - _run(agent, MagicMock(), device_controller.resize_device, a.get_id()) + _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) sizes[a.by_id_path] = 400 * _GIB - _run(agent, MagicMock(), device_controller.resize_device, a.get_id()) + _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) read = db.get_storage_device_by_id(a.get_id()) assert read.size == 400 * _GIB @@ -144,43 +146,43 @@ def test_second_growth_keeps_original_placement(self, db): class TestGrowCluster: - def test_uniform_growth_of_all_data_devices(self, db): + def test_mixed_growth_uses_the_smallest_factor(self, db): _seed_cluster(db) a, b = _aio_device("S1", 0), _aio_device("S2", 1) jm = _aio_device("J1", 2, size_gib=10, status=NVMeDevice.STATUS_JM) - _seed_node(db, [a, b, jm]) - agent = _agent({a.by_id_path: 200 * _GIB, b.by_id_path: 200 * _GIB}) + node = _seed_node(db, [a, b, jm]) + agent = _agent({a.by_id_path: 150 * _GIB, b.by_id_path: 200 * _GIB}) res = _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) assert res["errors"] == {} - assert res["uniform"] is True - assert sorted(r["device_id"] for r in res["devices"]) == sorted([a.get_id(), b.get_id()]) + assert res["grown"] is True + assert res["factor"] == 1.5 + ra, rb = db.get_storage_device_by_id(a.get_id()), db.get_storage_device_by_id(b.get_id()) + assert (ra.size, rb.size) == (150 * _GIB, 150 * _GIB) + assert (ra.unused_size(), rb.unused_size()) == (0, 50 * _GIB) + assert (ra.placement_size, rb.placement_size) == (100 * _GIB, 100 * _GIB) assert db.get_storage_device_by_id(jm.get_id()).size == 10 * _GIB - assert res["rebalance"] is None - assert db.get_storage_device_by_id(a.get_id()).placement_size == 100 * _GIB - def test_uneven_growth_reweights_and_rebalances(self, db): + fresh = db.get_storage_node_by_id(node.get_id()) + cl_map = distr_controller.get_distr_cluster_map([fresh], fresh, "distr-1") + weights = [i["weight"] for i in cl_map["map_prob"][0]["items"]] + assert weights == [100, 100] + + def test_shrunk_device_is_persisted_and_nothing_grows(self, db): _seed_cluster(db) a, b = _aio_device("S1", 0), _aio_device("S2", 1) - node = _seed_node(db, [a, b]) - agent = _agent({a.by_id_path: 200 * _GIB, b.by_id_path: 300 * _GIB}) - - with patch.object(device_controller.tasks_controller, "add_device_mig_task_for_node", - return_value=True) as queue: - res = _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) + _seed_node(db, [a, b]) + agent = _agent({a.by_id_path: 150 * _GIB, b.by_id_path: 80 * _GIB}) - assert res["uniform"] is False - assert res["rebalance"] == {"weights_updated": True, "started": True, "reason": ""} - queue.assert_called_once_with(node.get_id()) - assert db.get_storage_device_by_id(a.get_id()).placement_size == 200 * _GIB - assert db.get_storage_device_by_id(b.get_id()).placement_size == 300 * _GIB - assert db.get_cluster_by_id(CLUSTER_ID).reweight_pending is False + res = _run(agent, MagicMock(), device_controller.grow_cluster_devices, CLUSTER_ID) - fresh = db.get_storage_node_by_id(node.get_id()) - cl_map = distr_controller.get_distr_cluster_map([fresh], fresh, "distr-1") - weights = [i["weight"] for i in cl_map["map_prob"][0]["items"]] - assert weights == [200, 300] + assert res["grown"] is False + assert b.get_id() in res["errors"] + rb = db.get_storage_device_by_id(b.get_id()) + assert rb.size == 100 * _GIB + assert rb.is_shrunk() + assert db.get_storage_device_by_id(a.get_id()).size == 100 * _GIB def test_device_added_after_uniform_growth_is_weighted_on_the_same_scale(self, db): from simplyblock_core import storage_node_ops @@ -203,9 +205,10 @@ def test_failed_reopen_keeps_size_and_sets_unavailable(self, db): rpc = MagicMock() rpc.alceml_bdev_backend_resize_end.side_effect = RPCRemoteError("reopen", -7) - with pytest.raises(ValueError, match="unavailable"): - _run(agent, rpc, device_controller.resize_device, a.get_id()) + res = _run(agent, rpc, device_controller.grow_cluster_devices, CLUSTER_ID) + assert res["grown"] is False + assert "unavailable" in res["errors"][a.get_id()] read = db.get_storage_device_by_id(a.get_id()) assert read.size == 100 * _GIB assert read.placement_size == 0 diff --git a/tests/unit/test_lblk_device_resize.py b/tests/unit/test_lblk_device_resize.py index 6051dfa6d3..3e3063e71b 100644 --- a/tests/unit/test_lblk_device_resize.py +++ b/tests/unit/test_lblk_device_resize.py @@ -1,16 +1,18 @@ -"""Unit tests for the online growth of lblk devices (a LUN grown on its -storage array): device_controller.resize_device / grow_cluster_devices, the -frozen placement weight in distr_controller, and the host-side rescan in +"""Unit tests for the online growth of lblk devices (LUNs grown on their +storage array): device_controller.grow_cluster_devices, the frozen placement +weight in distr_controller, the capacity cap, and the host-side rescan in simplyblock_web.node_utils. Covered: - The cluster-map weight of a grown device stays at its placement size, - on full-map builds and on the add-device path. - - resize_device runs host rescan -> resize_begin -> bdev_aio_rescan -> - resize_end, persists size and placement_size, and reports uniformity. - - Refusals: not lblk, not online, partition, testing bdev, shrink. - - Recovery: a failed AIO rescan reopens alceml and leaves the device - unchanged; a failed reopen fails the I/O and sets the device unavailable. + on full-map builds and on the add-device path; growth never changes it. + - Growth is uniform: every data device grows by the smallest growth factor + of all block devices (1.5/2.0 -> 1.5); a device that did not grow stops + all growth; repeated runs change nothing. + - A shrunk block device is refused, logged, and raises device_shrunk. + - Refusals: not lblk, offline device or node, partition, testing bdev. + - Recovery: a failed reopen or AIO rescan leaves every size unchanged. + - The capacity statistics leave out the unused rest of a block device. - rescan_block_device: SCSI rescan, NVMe controller rescan, partitions. """ @@ -20,7 +22,7 @@ from unittest.mock import MagicMock, call, patch from simplyblock_core import constants, distr_controller, storage_node_ops, utils -from simplyblock_core.controllers import device_controller +from simplyblock_core.controllers import alerts_controller, device_controller from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.rpc_client import RPCRemoteError @@ -96,22 +98,20 @@ def test_add_device_weight_uses_placement_size(self): self.assertEqual(sent["devices"][0]["weight"], utils.convert_size(100 * _GIB, "GiB")) -class _ResizeHarness: - """Wires resize_device against mocks: one lblk cluster, one node.""" +class _GrowHarness: + """Wires grow_cluster_devices against mocks: one lblk cluster, one node + with data devices of 100 GiB whose block devices report the given sizes.""" - def __init__(self, dev, new_size, others=(), node_status=StorageNode.STATUS_ONLINE, - device_mode=constants.DEVICE_MODE_LBLK): - self.dev = dev - self.node = _node([dev, *others], status=node_status) + def __init__(self, sizes_gib, others=(), node_status=StorageNode.STATUS_ONLINE, + device_mode=constants.DEVICE_MODE_LBLK, dev_gib=100): + self.devs = [_aio_dev(f"dev-{i}", size_gib=dev_gib, order=i) for i in range(len(sizes_gib))] + self.node = _node([*self.devs, *others], status=node_status) self.cluster = MagicMock() self.cluster.device_mode = device_mode - self.cluster.status = "active" - self.cluster.reweight_pending = False self.rpc = MagicMock() self.node.rpc_client.return_value = self.rpc self.client = MagicMock() - self.client.rescan_block_device.return_value = ( - {"device": "sdb", "size": new_size, "rescanned": ["/sys/block/sdb/device/rescan"]}, None) + self.set_sizes(sizes_gib) self.node.client.return_value = self.client self.db = MagicMock() self.db.get_storage_device_by_id.side_effect = lambda _id: next( @@ -119,280 +119,231 @@ def __init__(self, dev, new_size, others=(), node_status=StorageNode.STATUS_ONLI self.db.get_storage_node_by_id.return_value = self.node self.db.get_storage_nodes_by_cluster_id.return_value = [self.node] self.db.get_cluster_by_id.return_value = self.cluster - self.db.get_job_tasks.return_value = [] - self.push_ok = True def _atomic(node, mut): mut(node) return node self.db.atomic_update.side_effect = _atomic - def run(self, fn=None, *args): + def set_sizes(self, sizes_gib): + by_path = {d.by_id_path: int(g * _GIB) for d, g in zip(self.devs, sizes_gib)} + self.client.rescan_block_device.side_effect = lambda path: ( + {"device": "sdx", "size": by_path[path], "rescanned": []}, None) + + def grow(self): + self.rpc.reset_mock() with patch.object(device_controller, "DBController", return_value=self.db), \ patch.object(device_controller.tasks_controller, "get_active_dev_restart_task", return_value=None), \ patch.object(device_controller, "device_set_unavailable") as self.set_unavail, \ patch.object(device_controller, "device_events") as self.events, \ - patch.object(device_controller.distr_controller, "send_cluster_map_to_node", - side_effect=lambda n: self.push_ok) as self.push, \ - patch.object(device_controller.tasks_controller, "add_device_mig_task_for_node", - return_value=True) as self.rebalance_task, \ patch.object(device_controller.time, "sleep"): - return (fn or device_controller.resize_device)(*(args or (self.dev.get_id(),))) - - -class TestResizeDevice(unittest.TestCase): - - def test_grows_device_and_freezes_weight(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB) - res = h.run() - self.assertTrue(res["resized"]) - self.assertEqual(res["old_size"], 100 * _GIB) - self.assertEqual(res["new_size"], 200 * _GIB) - self.assertEqual(h.dev.size, 200 * _GIB) - self.assertEqual(h.dev.placement_size, 100 * _GIB) - self.assertEqual(res["placement_size"], 100 * _GIB) - self.assertTrue(res["uniform"]) - h.client.rescan_block_device.assert_called_once_with("/dev/disk/by-id/wwn-dev-1") + return device_controller.grow_cluster_devices("cluster-1") + + def sizes(self): + return [d.size // _GIB for d in self.devs] + + +class TestGrowClusterDevices(unittest.TestCase): + + def test_uniform_growth_grows_every_device(self): + h = _GrowHarness([150, 150, 150]) + res = h.grow() + self.assertTrue(res["grown"]) + self.assertAlmostEqual(res["factor"], 1.5) + self.assertEqual(h.sizes(), [150, 150, 150]) + for d in h.devs: + self.assertEqual(d.placement_size, 100 * _GIB) + self.assertEqual(d.backend_size, 150 * _GIB) + self.assertEqual(d.unused_size(), 0) + self.assertEqual([r["unused_size"] for r in res["devices"]], [0, 0, 0]) self.assertEqual(h.rpc.mock_calls[:3], [ call.alceml_bdev_backend_resize_begin( - "alceml_dev-1", request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC), - call.bdev_aio_rescan("aio_dev-1"), + "alceml_dev-0", request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC), + call.bdev_aio_rescan("aio_dev-0"), call.alceml_bdev_backend_resize_end( - "alceml_dev-1", b_fail=False, cntr_path="", + "alceml_dev-0", b_fail=False, cntr_path="", request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC), ]) - h.events.device_resized.assert_called_once() - - def test_second_growth_keeps_original_placement(self): - h = _ResizeHarness(_aio_dev(size_gib=200, placement_gib=100), 300 * _GIB) - h.run() - self.assertEqual(h.dev.placement_size, 100 * _GIB) - self.assertEqual(h.dev.size, 300 * _GIB) - - def test_uneven_growth_warns(self): - other = _aio_dev("dev-2", size_gib=100, order=1) - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB, others=[other]) - res = h.run() - self.assertTrue(res["resized"]) - self.assertFalse(res["uniform"]) - self.assertIn("unevenly", res["warning"]) - - def test_even_growth_is_uniform(self): - other = _aio_dev("dev-2", size_gib=200, placement_gib=100, order=1) - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB, others=[other]) - self.assertTrue(h.run()["uniform"]) - - def test_journal_device_is_ignored_for_uniformity(self): - jm = _aio_dev("jm", size_gib=10, status=NVMeDevice.STATUS_JM, order=1) - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB, others=[jm]) - self.assertTrue(h.run()["uniform"]) - - def test_unchanged_size_is_noop(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 100 * _GIB) - res = h.run() - self.assertFalse(res["resized"]) - h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + self.assertEqual(h.rpc.alceml_bdev_backend_resize_begin.call_count, 3) + self.assertEqual(h.events.device_resized.call_count, 3) - def test_shrink_is_refused(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 50 * _GIB) - with self.assertRaisesRegex(ValueError, "shrank"): - h.run() - h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() - self.assertEqual(h.dev.size, 100 * _GIB) + def test_mixed_factors_grow_all_by_the_smallest(self): + h = _GrowHarness([150, 150, 150, 200, 200, 200]) + res = h.grow() + self.assertTrue(res["grown"]) + self.assertAlmostEqual(res["factor"], 1.5) + self.assertEqual(h.sizes(), [150] * 6) + self.assertEqual([d.unused_size() // _GIB for d in h.devs], [0, 0, 0, 50, 50, 50]) + self.assertEqual([d.placement_size // _GIB for d in h.devs], [100] * 6) + # Every alceml reopens: all of them take their whole block device. + self.assertEqual(h.rpc.alceml_bdev_backend_resize_begin.call_count, 6) + + def test_weights_never_change(self): + h = _GrowHarness([150, 200]) + before = TestPlacementWeight()._build_map(_node([copy_dev(d) for d in h.devs])) + h.grow() + after = TestPlacementWeight()._build_map(h.node) + self.assertEqual(before["map_prob"], after["map_prob"]) - def test_refuses_nvme_cluster(self): - h = _ResizeHarness(_aio_dev(), 200 * _GIB, device_mode=constants.DEVICE_MODE_NVME) - with self.assertRaisesRegex(ValueError, "lblk"): - h.run() - - def test_refuses_offline_device(self): - h = _ResizeHarness(_aio_dev(status=NVMeDevice.STATUS_UNAVAILABLE), 200 * _GIB) - with self.assertRaisesRegex(ValueError, "online"): - h.run() - - def test_refuses_offline_node(self): - h = _ResizeHarness(_aio_dev(), 200 * _GIB, node_status=StorageNode.STATUS_OFFLINE) - with self.assertRaisesRegex(ValueError, "Storage node must be online"): - h.run() - - def test_refuses_partition(self): - dev = _aio_dev() - dev.is_partition = True - h = _ResizeHarness(dev, 200 * _GIB) - with self.assertRaisesRegex(ValueError, "Partitions"): - h.run() - - def test_refuses_testing_bdev(self): - dev = _aio_dev() - dev.testing_bdev = "test_dev-1" - h = _ResizeHarness(dev, 200 * _GIB) - with self.assertRaisesRegex(ValueError, "testing or hang"): - h.run() - - def test_host_rescan_error(self): - h = _ResizeHarness(_aio_dev(), 200 * _GIB) - h.client.rescan_block_device.return_value = (None, "not found") - with self.assertRaisesRegex(ValueError, "Rescan"): - h.run() + def test_device_that_did_not_grow_stops_all_growth(self): + h = _GrowHarness([150, 100, 200]) + res = h.grow() + self.assertFalse(res["grown"]) + self.assertEqual(res["factor"], 1.0) + self.assertIn("Not every block device has grown", res["reason"]) + self.assertEqual(h.sizes(), [100, 100, 100]) h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + # The rescanned sizes are recorded all the same. + self.assertEqual([d.backend_size // _GIB for d in h.devs], [150, 100, 200]) - def test_aio_rescan_failure_reopens_and_leaves_device_unchanged(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB) - h.rpc.bdev_aio_rescan.side_effect = RPCRemoteError("rescan failed", -1) - with self.assertRaisesRegex(ValueError, "unchanged"): - h.run() - h.rpc.alceml_bdev_backend_resize_end.assert_called_once_with( - "alceml_dev-1", b_fail=False, cntr_path="", - request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC) - self.assertEqual(h.dev.size, 100 * _GIB) - self.assertEqual(h.dev.placement_size, 0) - h.set_unavail.assert_not_called() - - def test_reopen_failure_fails_io_and_sets_unavailable(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB) - h.rpc.alceml_bdev_backend_resize_end.side_effect = RPCRemoteError("reopen", -7) - with self.assertRaisesRegex(ValueError, "unavailable"): - h.run() - calls = h.rpc.alceml_bdev_backend_resize_end.call_args_list - self.assertEqual(len(calls), device_controller.RESIZE_END_ATTEMPTS + 1) - self.assertTrue(calls[-1].kwargs["b_fail"]) - h.set_unavail.assert_called_once() - self.assertEqual(h.dev.size, 100 * _GIB) - - def test_begin_failure_reopens(self): - h = _ResizeHarness(_aio_dev(size_gib=100), 200 * _GIB) - h.rpc.alceml_bdev_backend_resize_begin.side_effect = RPCRemoteError("busy", -6) - with self.assertRaisesRegex(ValueError, "resize_begin failed.*unchanged"): - h.run() - h.rpc.bdev_aio_rescan.assert_not_called() - h.rpc.alceml_bdev_backend_resize_end.assert_called() - - -class TestGrowClusterDevices(unittest.TestCase): - - def test_grows_all_aio_devices_and_collects_errors(self): - good = _aio_dev("dev-1", size_gib=100) - bad = _aio_dev("dev-2", size_gib=100, order=1) - bad.testing_bdev = "x" - jm = _aio_dev("jm", size_gib=10, status=NVMeDevice.STATUS_JM, order=2) - h = _ResizeHarness(good, 200 * _GIB, others=[bad, jm]) - res = h.run(device_controller.grow_cluster_devices, "cluster-1") - self.assertEqual([r["device_id"] for r in res["devices"]], ["dev-1"]) - self.assertIn("dev-2", res["errors"]) - self.assertNotIn("jm", res["errors"]) - - -class _GrowHarness(_ResizeHarness): - """Several devices whose block devices report their own new sizes.""" - - def __init__(self, sizes_gib, **kw): - devs = [_aio_dev(f"dev-{i}", size_gib=100, order=i) for i in range(len(sizes_gib))] - super().__init__(devs[0], 0, others=devs[1:], **kw) - self.devs = devs - by_path = {d.by_id_path: g * _GIB for d, g in zip(devs, sizes_gib)} - self.client.rescan_block_device.side_effect = lambda path: ( - {"device": "sdx", "size": by_path[path], "rescanned": []}, None) + def test_shrunk_device_is_refused_and_alerted(self): + h = _GrowHarness([150, 80, 150]) + res = h.grow() + self.assertFalse(res["grown"]) + self.assertIn("dev-1", res["errors"]) + self.assertIn("shrank", res["errors"]["dev-1"]) + h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + self.assertEqual(h.sizes(), [100, 100, 100]) + self.assertTrue(h.devs[1].is_shrunk()) + h.events.device_shrink_detected.assert_called_once() + alerts = alerts_controller._device_alerts("cluster-1", [h.node]) + shrunk = [a for a in alerts if a["kind"] == alerts_controller.DEVICE_SHRUNK] + self.assertEqual([a["device_id"] for a in shrunk], ["dev-1"]) + + def test_restored_lun_resolves_the_shrink(self): + h = _GrowHarness([100, 80]) + h.grow() + h.set_sizes([100, 100]) + res = h.grow() + self.assertFalse(h.devs[1].is_shrunk()) + self.assertEqual(res["errors"], {}) + alerts = alerts_controller._device_alerts("cluster-1", [h.node]) + self.assertFalse([a for a in alerts if a["kind"] == alerts_controller.DEVICE_SHRUNK]) - def grow(self, **kw): - return self.run(lambda cid: device_controller.grow_cluster_devices(cid, **kw), "cluster-1") + def test_repeated_runs_are_idempotent(self): + h = _GrowHarness([150, 200]) + h.grow() + sizes = h.sizes() + for _ in range(2): + res = h.grow() + self.assertFalse(res["grown"]) + self.assertEqual(h.sizes(), sizes) + h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + # A later, larger growth still applies on top. + h.set_sizes([300, 300]) + res = h.grow() + self.assertTrue(res["grown"]) + self.assertAlmostEqual(res["factor"], 2.0) + self.assertEqual(h.sizes(), [300, 300]) + self.assertEqual([d.placement_size // _GIB for d in h.devs], [100, 100]) + + def test_offline_device_stops_all_growth(self): + h = _GrowHarness([150, 150]) + h.devs[1].status = NVMeDevice.STATUS_UNAVAILABLE + res = h.grow() + self.assertFalse(res["grown"]) + self.assertIn("dev-1", res["errors"]) + h.client.rescan_block_device.assert_not_called() + self.assertEqual(h.sizes(), [100, 100]) + def test_journal_and_new_devices_are_ignored(self): + jm = _aio_dev("jm", size_gib=10, status=NVMeDevice.STATUS_JM, order=5) + new = _aio_dev("new", size_gib=50, status=NVMeDevice.STATUS_NEW, order=-1) + h = _GrowHarness([150, 150], others=[jm, new]) + res = h.grow() + self.assertTrue(res["grown"]) + self.assertEqual(jm.size, 10 * _GIB) + self.assertEqual(new.size, 50 * _GIB) -class TestRebalanceAfterGrowth(unittest.TestCase): + def test_refuses_nvme_cluster(self): + h = _GrowHarness([150], device_mode="nvme") + with self.assertRaisesRegex(ValueError, "lblk"): + h.grow() - def test_uniform_growth_keeps_weights_and_does_not_rebalance(self): - h = _GrowHarness([200, 200, 200]) - res = h.grow() - self.assertTrue(res["uniform"]) - self.assertIsNone(res["rebalance"]) - self.assertEqual([d.weight_size() for d in h.devs], [100 * _GIB] * 3) - h.push.assert_not_called() - h.rebalance_task.assert_not_called() - - def test_uneven_growth_reweights_pushes_and_rebalances(self): - h = _GrowHarness([200, 300, 100]) + def test_offline_node_stops_all_growth(self): + h = _GrowHarness([150], node_status=StorageNode.STATUS_OFFLINE) res = h.grow() - self.assertFalse(res["uniform"]) - self.assertEqual(res["rebalance"], {"weights_updated": True, "started": True, "reason": ""}) - self.assertEqual([d.weight_size() for d in h.devs], [200 * _GIB, 300 * _GIB, 100 * _GIB]) - h.push.assert_called_once_with(h.node) - h.rebalance_task.assert_called_once_with("node-1") - self.assertFalse(h.cluster.reweight_pending) - h.events.devices_reweighted.assert_called_once_with("cluster-1", True) - - def test_no_rebalance_keeps_frozen_weights(self): - h = _GrowHarness([200, 300]) - res = h.grow(rebalance=False) - self.assertFalse(res["uniform"]) - self.assertIsNone(res["rebalance"]) - self.assertEqual([d.weight_size() for d in h.devs], [100 * _GIB] * 2) - h.push.assert_not_called() - - def test_open_migration_blocks_reweight(self): - h = _GrowHarness([200, 300]) - task = MagicMock(function_name=device_controller.JobSchedule.FN_NEW_DEV_MIG, - status="running", canceled=False, uuid="t-1") - h.db.get_job_tasks.return_value = [task] + self.assertFalse(res["grown"]) + self.assertIn("Storage node must be online", res["errors"]["dev-0"]) + + def test_partition_or_testing_bdev_stops_all_growth(self): + h = _GrowHarness([150, 150]) + h.devs[0].is_partition = True + h.devs[1].testing_bdev = "x" res = h.grow() - self.assertFalse(res["rebalance"]["started"]) - self.assertIn("t-1", res["rebalance"]["reason"]) - self.assertEqual([d.weight_size() for d in h.devs], [100 * _GIB] * 2) - self.assertFalse(h.cluster.reweight_pending) - h.push.assert_not_called() - - def test_inactive_cluster_blocks_reweight(self): - h = _GrowHarness([200, 300]) - h.cluster.status = "degraded" + self.assertFalse(res["grown"]) + self.assertEqual(set(res["errors"]), {"dev-0", "dev-1"}) + + def test_rescan_error_stops_all_growth(self): + h = _GrowHarness([150, 150]) + h.client.rescan_block_device.side_effect = lambda path: (None, "no such device") res = h.grow() - self.assertIn("degraded", res["rebalance"]["reason"]) - self.assertEqual([d.weight_size() for d in h.devs], [100 * _GIB] * 2) + self.assertFalse(res["grown"]) + h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() - def test_failed_growth_does_not_reweight(self): - h = _GrowHarness([200, 300]) - h.devs[1].testing_bdev = "x" + def test_reopen_failure_keeps_all_sizes_and_sets_unavailable(self): + h = _GrowHarness([150, 150]) + h.rpc.alceml_bdev_backend_resize_end.side_effect = [ + None, RPCRemoteError("reopen", -7), RPCRemoteError("reopen", -7), + RPCRemoteError("reopen", -7), None] res = h.grow() + self.assertFalse(res["grown"]) self.assertIn("dev-1", res["errors"]) - self.assertFalse(res["rebalance"]["started"]) - self.assertIn("failed to grow", res["rebalance"]["reason"]) - h.push.assert_not_called() + self.assertEqual(h.sizes(), [100, 100]) + h.set_unavail.assert_called_once() + h.events.device_resized.assert_not_called() - def test_failed_push_keeps_pending_and_retry_finishes(self): - h = _GrowHarness([200, 300]) - h.push_ok = False + def test_aio_rescan_failure_keeps_all_sizes(self): + h = _GrowHarness([150, 150]) + h.rpc.bdev_aio_rescan.side_effect = RPCRemoteError("rescan", -5) res = h.grow() - self.assertFalse(res["rebalance"]["started"]) - self.assertIn("push failed", res["rebalance"]["reason"]) - self.assertTrue(h.cluster.reweight_pending) - self.assertEqual([d.weight_size() for d in h.devs], [200 * _GIB, 300 * _GIB]) - h.rebalance_task.assert_not_called() - - # Re-run: sizes unchanged, growth now looks uniform, but the pending - # flag makes it push again and start the rebalance. - h.push_ok = True + self.assertFalse(res["grown"]) + self.assertIn("unchanged", res["errors"]["dev-0"]) + self.assertEqual(h.sizes(), [100, 100]) + h.set_unavail.assert_not_called() + + def test_begin_failure_reopens(self): + h = _GrowHarness([150]) + h.rpc.alceml_bdev_backend_resize_begin.side_effect = RPCRemoteError("timeout", -1) res = h.grow() - self.assertTrue(res["rebalance"]["started"]) - self.assertFalse(h.cluster.reweight_pending) - h.rebalance_task.assert_called_once_with("node-1") - - def test_weights_updated_without_volume_stores_to_rebalance(self): - h = _GrowHarness([200, 300]) - - def _grow_without_queue(cluster_id): - h.rebalance_task.return_value = False - return device_controller.grow_cluster_devices(cluster_id) - res = h.run(_grow_without_queue, "cluster-1") - self.assertTrue(res["rebalance"]["weights_updated"]) - self.assertFalse(res["rebalance"]["started"]) - self.assertIn("no rebalance was queued", res["rebalance"]["reason"]) - self.assertFalse(h.cluster.reweight_pending) - self.assertEqual([d.weight_size() for d in h.devs], [200 * _GIB, 300 * _GIB]) - - def test_cluster_map_carries_new_weights_after_reweight(self): - h = _GrowHarness([200, 300]) - h.grow() - cl_map = TestPlacementWeight._build_map(None, h.node) - weights = [i["weight"] for i in cl_map["map_prob"][0]["items"]] - self.assertEqual(weights, [200, 300]) + self.assertFalse(res["grown"]) + h.rpc.alceml_bdev_backend_resize_end.assert_called_with( + "alceml_dev-0", b_fail=False, cntr_path="", + request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC) + h.set_unavail.assert_not_called() + + +class TestUnusedCapacity(unittest.TestCase): + """The capacity statistics leave out the part of a block device beyond + the device size: alceml spans the whole block device.""" + + def test_device_stats_cap_total_at_device_size(self): + from simplyblock_core.models.stats import DeviceStatObject + from simplyblock_core.services import capacity_and_stats_collector as collector + dev = _aio_dev(size_gib=150, placement_gib=100) + dev.backend_size = 200 * _GIB + cl = MagicMock() + cl.get_id.return_value = "cluster-1" + cl.distr_ndcs, cl.distr_npcs = 1, 1 + page = 2 * 1024 * 1024 + cap = {"res": 1, "npages_nmax": (200 * _GIB) // page, "npages_used": 0, "pba_page_size": page} + db = MagicMock() + db.get_device_stats.return_value = [] + with patch.object(collector, "db", db), patch.object(collector, "last_object_record", {}), \ + patch.object(DeviceStatObject, "write_to_db", lambda self, kv: True): + rec = collector.add_device_stats(cl, dev, cap, None) + self.assertEqual(rec.size_total_raw, 150 * _GIB) + + def test_alert_free_device_reports_unused(self): + dev = _aio_dev(size_gib=150, placement_gib=100) + dev.backend_size = 200 * _GIB + self.assertEqual(dev.unused_size(), 50 * _GIB) + self.assertFalse(dev.is_shrunk()) + + +def copy_dev(d): + return NVMeDevice(d.to_dict()) class TestScaledPlacementSize(unittest.TestCase): @@ -411,7 +362,7 @@ def test_uniform_growth_scales_new_device(self): _aio_dev("b", size_gib=200, placement_gib=100, order=1)]) self.assertEqual(storage_node_ops.scaled_placement_size(db, "cluster-1", 200 * _GIB), 100 * _GIB) - def test_uneven_growth_uses_full_size(self): + def test_inconsistent_ratios_use_full_size(self): db = self._db([_aio_dev("a", size_gib=200, placement_gib=100), _aio_dev("b", size_gib=100, order=1)]) self.assertEqual(storage_node_ops.scaled_placement_size(db, "cluster-1", 200 * _GIB), 0) diff --git a/tests/unit/web/api/v2/test_device_endpoints.py b/tests/unit/web/api/v2/test_device_endpoints.py index b42cdef191..7d87b24725 100644 --- a/tests/unit/web/api/v2/test_device_endpoints.py +++ b/tests/unit/web/api/v2/test_device_endpoints.py @@ -374,82 +374,67 @@ def test_detail_dispatches_watch_device(self, client, device, device_controller, CLUSTER_ID, STORAGE_NODE_ID, DEVICE_ID) -RESIZE_RESULT = { - 'device_id': DEVICE_ID, 'old_size': 100, 'new_size': 200, 'placement_size': 100, - 'resized': True, 'uniform': False, 'warning': 'grown unevenly', +GROWN_DEVICE = { + 'device_id': DEVICE_ID, 'old_size': 100, 'new_size': 150, 'backend_size': 200, + 'unused_size': 50, 'placement_size': 100, } -class TestResizeDevice: - - def test_returns_resize_result(self, client, db, device, device_controller): - device_controller.resize_device.return_value = RESIZE_RESULT - - response = client.post(f'{BASE}/{DEVICE_ID}/resize') - - assert response.status_code == 200 - assert response.json() == RESIZE_RESULT - device_controller.resize_device.assert_called_once_with(DEVICE_ID) - - def test_refusal_returns_409(self, client, db, device, device_controller): - device_controller.resize_device.side_effect = ValueError('shrank') +class TestResizeDeviceRemoved: + def test_single_device_resize_is_gone(self, client, db, device, device_controller): response = client.post(f'{BASE}/{DEVICE_ID}/resize') - assert response.status_code == 409 - assert 'shrank' in response.json()['detail'] + assert response.status_code in (404, 405) class TestGrowClusterDevices: URL = f'/api/v2/clusters/{CLUSTER_ID}/grow-devices' - def test_grows_devices_of_lblk_cluster(self, client, db, cluster, monkeypatch): + def _mock(self, monkeypatch, result): from unittest.mock import MagicMock import simplyblock_web.api.v2.cluster as cluster_module mock = MagicMock() - mock.grow_cluster_devices.return_value = {'devices': [RESIZE_RESULT], 'errors': {}, 'uniform': True} + mock.grow_cluster_devices.return_value = result monkeypatch.setattr(cluster_module, 'device_controller', mock) + return mock + + def test_grows_devices_of_lblk_cluster(self, client, db, cluster, monkeypatch): + mock = self._mock(monkeypatch, { + 'grown': True, 'factor': 1.5, 'devices': [GROWN_DEVICE], 'errors': {}, 'reason': ''}) cluster.device_mode = 'lblk' response = client.post(self.URL) assert response.status_code == 200 - assert response.json()['uniform'] is True - assert response.json()['rebalance'] is None - mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID, rebalance=True) - - def test_uneven_growth_reports_rebalance(self, client, db, cluster, monkeypatch): - from unittest.mock import MagicMock - - import simplyblock_web.api.v2.cluster as cluster_module - mock = MagicMock() - mock.grow_cluster_devices.return_value = { - 'devices': [RESIZE_RESULT], 'errors': {}, 'uniform': False, - 'rebalance': {'weights_updated': True, 'started': True, 'reason': ''}} - monkeypatch.setattr(cluster_module, 'device_controller', mock) + body = response.json() + assert body['grown'] is True + assert body['factor'] == 1.5 + assert body['devices'] == [GROWN_DEVICE] + mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID) + + def test_nothing_grown_reports_reason_and_errors(self, client, db, cluster, monkeypatch): + self._mock(monkeypatch, { + 'grown': False, 'factor': 1.0, 'devices': [], 'errors': {DEVICE_ID: 'shrank'}, + 'reason': 'A block device is smaller than its device; nothing was grown'}) cluster.device_mode = 'lblk' response = client.post(self.URL) assert response.status_code == 200 - assert response.json()['rebalance'] == {'weights_updated': True, 'started': True, 'reason': ''} + assert response.json()['grown'] is False + assert response.json()['errors'] == {DEVICE_ID: 'shrank'} - def test_rebalance_can_be_disabled(self, client, db, cluster, monkeypatch): - from unittest.mock import MagicMock - - import simplyblock_web.api.v2.cluster as cluster_module - mock = MagicMock() - mock.grow_cluster_devices.return_value = { - 'devices': [], 'errors': {}, 'uniform': False, 'rebalance': None} - monkeypatch.setattr(cluster_module, 'device_controller', mock) + def test_rebalance_parameter_is_gone(self, client, db, cluster, monkeypatch): + mock = self._mock(monkeypatch, { + 'grown': False, 'factor': 1.0, 'devices': [], 'errors': {}, 'reason': 'x'}) cluster.device_mode = 'lblk' - response = client.post(self.URL, params={'rebalance': 'false'}) + client.post(self.URL, params={'rebalance': 'false'}) - assert response.status_code == 200 - mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID, rebalance=False) + mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID) def test_nvme_cluster_returns_409(self, client, db, cluster): cluster.device_mode = 'nvme' From b4afb765f5f2402aeb2cdd6b96c7513e0d0333a3 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 30 Oct 2026 16:55:34 +0100 Subject: [PATCH 02/50] fix(lblk): grow-devices lists shrunk devices separately Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/controllers/device_controller.py | 11 +++++++---- simplyblock_web/api/v2/cluster/__init__.py | 5 +++-- tests/unit/test_lblk_device_resize.py | 1 + 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index 1ad963275b..d8317fd5f6 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -2064,9 +2064,9 @@ def _reopen_alceml(snode, dev): raise _resize_failed(device_id, dev, f"AIO rescan failed: {rescan_error}", False) -def _growth_result(grown, factor, devices, errors, reason): +def _growth_result(grown, factor, devices, errors, reason, shrunk=()): return {"grown": grown, "factor": round(factor, 6), "devices": devices, - "errors": errors, "reason": reason} + "errors": errors, "reason": reason, "shrunk": list(shrunk)} def _device_report(dev, old_size): @@ -2089,7 +2089,8 @@ def grow_cluster_devices(cluster_id): finishes the job. Repeated runs without new growth change nothing. Returns ``{"grown": bool, "factor": float, "devices": [...], - "errors": {device_id: message}, "reason": str}``; every device entry has + "errors": {device_id: message}, "reason": str, "shrunk": [device_id]}``; + every device entry has ``device_id``, ``old_size``, ``new_size``, ``backend_size``, ``unused_size`` and ``placement_size``.""" db_controller = DBController() @@ -2135,7 +2136,9 @@ def grow_cluster_devices(cluster_id): f"The block device shrank to {dev.backend_size} bytes, below the device size of " f"{dev.size} bytes. Shrinking is not supported: restore the LUN to at least the " f"device size") - return _growth_result(False, 1.0, [], errors, "A block device is smaller than its device; nothing was grown") + return _growth_result(False, 1.0, [], errors, + "A block device is smaller than its device; nothing was grown", + shrunk=[dev.get_id() for _, dev in shrunk]) factor = min(backend[dev.get_id()] / dev.size for _, dev in pairs) if factor < GROWTH_MIN_FACTOR: diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index 7a134e729e..0220081507 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -347,6 +347,7 @@ class DeviceGrowthDTO(BaseModel): devices: builtins.list[DeviceGrowthEntryDTO] errors: dict[str, str] reason: str + shrunk: builtins.list[str] = [] @instance_api.post('/grow-devices', name='clusters:grow_devices', response_model=DeviceGrowthDTO) @@ -358,8 +359,8 @@ def grow_devices(cluster: Cluster) -> DeviceGrowthDTO: of all of them; the rest of a larger block device stays unused (`unused_size`). If one block device did not grow, nothing grows (`grown` false, `reason`). Device weights and data placement never change. - A block device smaller than its device is refused, reported in `errors`, - and raises the device_shrunk alert. + A block device smaller than its device is refused, listed in `shrunk` + and `errors`, and raises the device_shrunk alert. """ if cluster.device_mode != constants.DEVICE_MODE_LBLK: raise HTTPException(409, 'Online device growth needs a cluster in lblk device mode') diff --git a/tests/unit/test_lblk_device_resize.py b/tests/unit/test_lblk_device_resize.py index 3e3063e71b..4e33617bf9 100644 --- a/tests/unit/test_lblk_device_resize.py +++ b/tests/unit/test_lblk_device_resize.py @@ -203,6 +203,7 @@ def test_shrunk_device_is_refused_and_alerted(self): self.assertFalse(res["grown"]) self.assertIn("dev-1", res["errors"]) self.assertIn("shrank", res["errors"]["dev-1"]) + self.assertEqual(res["shrunk"], ["dev-1"]) h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() self.assertEqual(h.sizes(), [100, 100, 100]) self.assertTrue(h.devs[1].is_shrunk()) From c96c16200b3d35a0db178f0def7dc8c4a154cd1d Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 9 Nov 2026 14:58:30 +0100 Subject: [PATCH 03/50] feat(floors): enforce the erasure-coding resource floors Product rules of 2026-11-09 for an n+k cluster: >= 1 data device per node, >= n+2k online data devices (n+k with the new raid_devices declaration), >= n+k failure domains when they are used, >= n+2k nodes; journal devices never count. Single-node (1 node, n+k devices) and two-node (2 nodes) clusters keep their topology overrides. - simplyblock_core/floors.py: Floors, cluster_floors, count_data_devices, floor_violations, evaluate. - Cluster.raid_devices (create-time, immutable) with `cluster create/add --raid-devices` and the v1/v2 API; --strict-node-anti-affinity help no longer claims n+k+1 nodes. - Fresh activation refuses below any floor (a node without data devices included); re-activation keeps the old device gate. The fresh FD activation needs max(k+2, n+k) domains. - Volume creation refuses below the floors of the volume's scheme. - Planned shutdown (force overrides) and node removal refuse when the cluster without the node is below a floor; the rebalancing headroom formula is gone. - Monitor: DEGRADED (never SUSPENDED) while online resources are below a floor, with a FLOOR cluster event naming it; suspension rules unchanged. Co-Authored-By: Claude Opus 5.5 --- simplyblock_cli/cli-reference.yaml | 26 +- simplyblock_cli/cli.py | 6 +- simplyblock_cli/clibase.py | 4 + simplyblock_core/cluster_ops.py | 52 ++- .../controllers/cluster_events.py | 28 ++ .../controllers/cluster_expansion/planner.py | 24 +- .../controllers/events_controller.py | 3 + .../controllers/lvol_controller.py | 18 +- simplyblock_core/floors.py | 251 +++++++++++ simplyblock_core/models/cluster.py | 7 + .../services/storage_node_monitor.py | 48 ++- simplyblock_core/storage_node_ops.py | 60 +-- simplyblock_web/api/v1/cluster.py | 2 + simplyblock_web/api/v2/_dtos.py | 2 + simplyblock_web/api/v2/cluster/__init__.py | 3 + tests/unit/test_activation_growth_guard.py | 9 +- tests/unit/test_cluster_floors.py | 389 ++++++++++++++++++ tests/unit/test_failure_domain.py | 4 + tests/unit/test_ftt_protection.py | 286 +++++-------- tests/unit/test_node_removal.py | 12 + tests/unit/test_removal_three_steps.py | 4 +- tests/unit/test_sync_replication_site_loss.py | 9 + 22 files changed, 1023 insertions(+), 224 deletions(-) create mode 100644 simplyblock_core/floors.py create mode 100644 tests/unit/test_cluster_floors.py diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 964cfb0b76..d29a4f7627 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1032,11 +1032,20 @@ commands: private: true - name: "--strict-node-anti-affinity" help: > - Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This - requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster. + Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The + cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ + nodes); the flag only affects status reporting. dest: strict_node_anti_affinity type: bool action: store_true + - name: "--raid-devices" + help: > + Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers + the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no + other device is writable. Cannot be changed after cluster creation. + dest: raid_devices + type: bool + action: store_true - name: "--enable-failure-domain" help: "Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed." dest: enable_failure_domain @@ -1300,11 +1309,20 @@ commands: private: true - name: "--strict-node-anti-affinity" help: > - Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. - This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster." + Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The + cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ + nodes); the flag only affects status reporting. dest: strict_node_anti_affinity type: bool action: store_true + - name: "--raid-devices" + help: > + Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers + the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no + other device is writable. Cannot be changed after cluster creation. + dest: raid_devices + type: bool + action: store_true - name: "--enable-failure-domain" help: "Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed." dest: enable_failure_domain diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 36efcec0bb..dd84160844 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -466,7 +466,8 @@ def init_cluster__create(self, subparser): subcommand.add_argument('--inflight-io-threshold', help='The number of inflight IOs allowed before the IO queuing starts. Default: `4`.', type=int, default=4, dest='inflight_io_threshold') if self.developer_mode: subcommand.add_argument('--disable-monitoring', help='Disable monitoring stack, false by default. Default: `false`.', dest='disable_monitoring', action='store_true') - subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster.', dest='strict_node_anti_affinity', action='store_true') + subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ nodes); the flag only affects status reporting.', dest='strict_node_anti_affinity', action='store_true') + subcommand.add_argument('--raid-devices', help='Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no other device is writable. Cannot be changed after cluster creation.', dest='raid_devices', action='store_true') subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') subcommand.add_argument('--sync-replication', help='Enable site-level synchronous replication over exactly two sites. Each storage node must then be added with --site; every lvstore is replicated to the other site. Requires --ha-type ha. Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='sync_replication', action='store_true') subcommand.add_argument('--device-mode', help='Storage-device mode for the whole cluster. \'nvme\' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. \'lblk\': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at \'sn configure\' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected.', type=str, default='nvme', dest='device_mode', choices=['nvme','lblk',]) @@ -513,7 +514,8 @@ def init_cluster__add(self, subparser): subcommand.add_argument('--max-queue-size', help='The max size the queue will grow. Default: `128`.', type=int, default=128, dest='max_queue_size') if self.developer_mode: subcommand.add_argument('--inflight-io-threshold', help='The number of inflight IOs allowed before the IO queuing starts. Default: `4`.', type=int, default=4, dest='inflight_io_threshold') - subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. This requires a minimum of _data-chunks-in-stripe + parity-chunks-in-stripe + 1_ nodes in the cluster."', dest='strict_node_anti_affinity', action='store_true') + subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ nodes); the flag only affects status reporting.', dest='strict_node_anti_affinity', action='store_true') + subcommand.add_argument('--raid-devices', help='Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no other device is writable. Cannot be changed after cluster creation.', dest='raid_devices', action='store_true') subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') subcommand.add_argument('--sync-replication', help='Enable site-level synchronous replication over exactly two sites. Each storage node must then be added with --site; every lvstore is replicated to the other site. Requires --ha-type ha. Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='sync_replication', action='store_true') subcommand.add_argument('--device-mode', help='Storage-device mode for the whole cluster. \'nvme\' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. \'lblk\': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at \'sn configure\' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected.', type=str, default='nvme', dest='device_mode', choices=['nvme','lblk',]) diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 15d1ff1e92..73b3d9d8d5 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -1937,6 +1937,7 @@ def cluster_add(self, args): max_queue_size = args.max_queue_size inflight_io_threshold = args.inflight_io_threshold strict_node_anti_affinity = args.strict_node_anti_affinity + raid_devices = getattr(args, 'raid_devices', False) is_single_node = args.is_single_node client_data_nic = args.client_data_nic enable_failure_domain = getattr(args, 'enable_failure_domain', False) @@ -1966,6 +1967,7 @@ def cluster_add(self, args): sync_replication=getattr(args, 'sync_replication', False), page_copy=getattr(args, 'page_copy', False), md_journal=md_journal, + raid_devices=raid_devices, ) def cluster_create(self, args): @@ -1995,6 +1997,7 @@ def cluster_create(self, args): inflight_io_threshold = args.inflight_io_threshold disable_monitoring = args.disable_monitoring strict_node_anti_affinity = args.strict_node_anti_affinity + raid_devices = getattr(args, 'raid_devices', False) name = args.name tls_secret = args.tls_secret ingress_host_source = args.ingress_host_source @@ -2043,6 +2046,7 @@ def cluster_create(self, args): sync_replication=getattr(args, 'sync_replication', False), page_copy=getattr(args, 'page_copy', False), md_journal=md_journal, + raid_devices=raid_devices, ) def query_yes_no(self, question, default="yes"): diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 05f9689d25..53593fca09 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -26,6 +26,7 @@ import docker from simplyblock_core import ( constants, + floors, index_ops, jm_raid, mgmt_node_ops, @@ -48,6 +49,9 @@ from simplyblock_core.controllers.cluster_expansion.planner import ( RemoteTripletPlacementError, ) +from simplyblock_core.controllers.cluster_expansion.planner import ( + fd_activation_min_domains as fd_planner_min_domains, +) from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import ( SyncReplicationSiteError, @@ -400,6 +404,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, sync_replication=False, page_copy=False, md_journal=True, + raid_devices=False, ) -> str: if (distr_ndcs, distr_npcs) not in SUPPORTED_ERASURE_CODING_SCHEMES: raise ValueError("Unsupported erasure coding scheme") @@ -564,6 +569,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.atomic_4k = bool(atomic_4k) cluster.page_copy = bool(page_copy) cluster.md_journal = bool(md_journal) + cluster.raid_devices = bool(raid_devices) cluster.client_data_nic = client_data_nic or "" cluster.max_fault_tolerance = max_fault_tolerance cluster.nvmf_base_port = nvmf_base_port @@ -681,6 +687,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn leader_site="", page_copy=False, md_journal=True, + raid_devices=False, snode_api_service=constants.K8S_SNODE_API_SERVICE, spdk_proxy_service=constants.K8S_SPDK_PROXY_SERVICE, spdk_proxy_tls_secret=constants.K8S_SPDK_PROXY_TLS_SECRET, @@ -718,6 +725,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn leader_site=leader_site, page_copy=page_copy, md_journal=md_journal, + raid_devices=raid_devices, snode_api_service=snode_api_service, spdk_proxy_service=spdk_proxy_service, spdk_proxy_tls_secret=spdk_proxy_tls_secret, @@ -755,6 +763,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca leader_site="", page_copy=False, md_journal=True, + raid_devices=False, snode_api_service=constants.K8S_SNODE_API_SERVICE, spdk_proxy_service=constants.K8S_SPDK_PROXY_SERVICE, spdk_proxy_tls_secret=constants.K8S_SPDK_PROXY_TLS_SECRET, @@ -899,6 +908,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca cluster.atomic_4k = bool(atomic_4k) cluster.page_copy = bool(page_copy) cluster.md_journal = bool(md_journal) + cluster.raid_devices = bool(raid_devices) cluster.client_data_nic = client_data_nic or "" cluster.max_fault_tolerance = max_fault_tolerance cluster.nvmf_base_port = nvmf_base_port @@ -1392,12 +1402,23 @@ def _record_activation_topology(cluster, single_node, fresh) -> None: def activation_minimum_devices(cluster, single_node_cluster) -> int: - """ndcs+npcs devices are needed for placement; the +1 spare is rebuild - headroom that a single-node cluster (no data redundancy to rebuild onto - a spare) does not require.""" + """The RE-activation device gate (recovery of an existing layout): ndcs+npcs + devices are needed for placement; the +1 spare is rebuild headroom that a + single-node cluster (no data redundancy to rebuild onto a spare) does not + require. A FRESH activation is judged by the full floors instead + (``activation_floor_violations``); a re-activation deliberately keeps + this weaker gate so a drifted cluster can still be recovered.""" return cluster.distr_ndcs + cluster.distr_npcs + (0 if single_node_cluster else 1) +def activation_floor_violations(cluster, snodes, online_nodes) -> builtins.list[str]: + """The floors (simplyblock_core.floors) a fresh activation would violate + with ``online_nodes`` online: node count, one data device per node, the + cluster-wide data-device count and, with failure domains, the domain + count. Journal devices never count (they sit in ``jm_device``).""" + return floors.evaluate(cluster, snodes, online_nodes=online_nodes) + + #: Sites of a sync-replication cluster: one simplyblock cluster spans exactly two. SYNC_SITE_COUNT = 2 #: Distinct hosts a site needs for a host-disjoint primary/secondary/tertiary triplet. @@ -1630,10 +1651,23 @@ def _revert() -> None: logger.warning("Single-node cluster: activating as non-HA " "(no secondary nodes, single journal) regardless of ha_type") - minimum_devices = activation_minimum_devices(cluster, single_node_cluster) - if dev_count < minimum_devices: - _revert() - raise ValueError(f"Failed to activate cluster, No enough online device.. Minimum is {minimum_devices}") + if is_fresh_activation: + # Fresh activation refuses below any floor (simplyblock_core.floors). + # A node without a data device is refused here too: downstream, + # _prepare_cluster_devices_jm_on_dev accepts an empty device list. + floor_violations = activation_floor_violations(cluster, snodes, online_nodes) + if floor_violations: + _revert() + raise ValueError("Failed to activate cluster, below the resource floors for " + f"the {cluster.distr_ndcs}+{cluster.distr_npcs} scheme: " + + "; ".join(floor_violations)) + else: + # Re-activation (recovery): the weaker device gate of before, so a + # drifted or partially-online cluster can still be brought back. + minimum_devices = activation_minimum_devices(cluster, single_node_cluster) + if dev_count < minimum_devices: + _revert() + raise ValueError(f"Failed to activate cluster, No enough online device.. Minimum is {minimum_devices}") # Sync replication: the configured two-site layout is enforced on every # activation (force included) -- a third site or an unsited node is never @@ -1679,7 +1713,7 @@ def _revert() -> None: fd_desired_layout: dict[str, tuple[str, str]] = {} if cluster.enable_failure_domain: distinct_domains = {node.failure_domain for node in online_nodes if node.failure_domain >= 0} - min_domains = cluster.distr_npcs + 2 + min_domains = fd_planner_min_domains(cluster.distr_npcs, cluster.distr_ndcs) if len(distinct_domains) < min_domains: logger.warning( "Failure-domain feature is enabled but only %d distinct failure " @@ -1736,7 +1770,7 @@ def _on_site(msg: str, site=site) -> str: # stranding another node's secondary/tertiary with none at all. # This also subsumes the plain "at least two domains" floor. domain_count_violation = fd_planner.fd_activation_domain_count_violation( - cluster.distr_npcs, len(fd_host_counts)) + cluster.distr_npcs, len(fd_host_counts), ndcs=cluster.distr_ndcs) if domain_count_violation: _fd_fail(_on_site(domain_count_violation)) if len(set(fd_host_counts.values())) != 1: diff --git a/simplyblock_core/controllers/cluster_events.py b/simplyblock_core/controllers/cluster_events.py index 6380f05c48..6206784da3 100644 --- a/simplyblock_core/controllers/cluster_events.py +++ b/simplyblock_core/controllers/cluster_events.py @@ -71,6 +71,34 @@ def _cluster_cap_event(cluster, msg, event_level): event_level=event_level) +def cluster_floor_violation(cluster, violations): + """The monitor found the online resources below a floor (the cluster is + DEGRADED for it); ``violations`` are the floor texts of + simplyblock_core.floors.floor_violations. One event per change of the set.""" + msg = "Cluster below resource floor: " + "; ".join(violations) + return ec.log_event_cluster( + cluster_id=cluster.get_id(), + node_id=cluster.get_id(), + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_FLOOR, + db_object=cluster, + caused_by=ec.CAUSED_BY_MONITOR, + message=msg, + event_level=EventObj.LEVEL_WARN) + + +def cluster_floor_restored(cluster): + return ec.log_event_cluster( + cluster_id=cluster.get_id(), + node_id=cluster.get_id(), + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_FLOOR, + db_object=cluster, + caused_by=ec.CAUSED_BY_MONITOR, + message="Cluster back at or above every resource floor", + event_level=EventObj.LEVEL_INFO) + + def cluster_cap_warn(cluster, util): msg = f"Cluster absolute capacity reached: {util}%" return _cluster_cap_event(cluster, msg, event_level=EventObj.LEVEL_WARN) diff --git a/simplyblock_core/controllers/cluster_expansion/planner.py b/simplyblock_core/controllers/cluster_expansion/planner.py index fa1f8d5b76..597dc84db5 100644 --- a/simplyblock_core/controllers/cluster_expansion/planner.py +++ b/simplyblock_core/controllers/cluster_expansion/planner.py @@ -503,11 +503,24 @@ def fd_balance_violation( return None +def fd_activation_min_domains(npcs: int, ndcs: int = 0) -> int: + """The failure domains a fresh activation needs: the rotation's spare + rule (npcs+2, see fd_activation_domain_count_violation) and the EC floor + of ndcs+npcs domains (simplyblock_core.floors, floor 3), whichever is + higher.""" + return max(npcs + 2, ndcs + npcs) + + def fd_activation_domain_count_violation( - npcs: int, distinct_domain_count: int, + npcs: int, distinct_domain_count: int, ndcs: int = 0, ) -> str | None: """Validate the number of distinct failure domains for fresh activation. + Two rules, the higher one binds (``fd_activation_min_domains``): the EC + floor of ``ndcs+npcs`` domains -- a stripe's chunks must each have a + domain of their own when failure domains are used at all -- and the + rotation-spare rule below. + A 2-FD layout can never absorb a second independent failure once one domain is fully down (confirmed with the backend team), so it is not supported at any npcs level. @@ -533,12 +546,13 @@ def fd_activation_domain_count_violation( human-readable reason on violation, ``None`` when the count is acceptable. """ - min_domains = npcs + 2 + min_domains = fd_activation_min_domains(npcs, ndcs) if distinct_domain_count < min_domains: + rule = ("n+k" if ndcs + npcs >= npcs + 2 else "k+2") return ( - f"failure domains are enabled with npcs={npcs}, which requires at " - f"least {min_domains} distinct failure domains (2 domains is not " - f"supported at any npcs level); currently have " + f"failure domains are enabled with ndcs={ndcs}, npcs={npcs}, which " + f"requires at least {min_domains} distinct failure domains ({rule}; " + f"2 domains is not supported at any npcs level); currently have " f"{distinct_domain_count}. Add hosts in additional domains, or " f"disable failure domains, then activate.") return None diff --git a/simplyblock_core/controllers/events_controller.py b/simplyblock_core/controllers/events_controller.py index 04b3ac0757..611ddbf4e0 100644 --- a/simplyblock_core/controllers/events_controller.py +++ b/simplyblock_core/controllers/events_controller.py @@ -13,6 +13,9 @@ EVENT_OBJ_CREATED = "OBJ_CREATED" EVENT_OBJ_DELETED = "OBJ_DELETED" EVENT_CAPACITY = "CAPACITY" +# The cluster's online resources are below an erasure-coding floor +# (simplyblock_core.floors); the message names the floor. +EVENT_FLOOR = "FLOOR" EVENT_LIMIT_REACHED = "LIMIT_REACHED" #: A distrib reported an IO it could not serve from the chunks it can reach. EVENT_DISTR_IO_ERROR = "DISTR_IO_ERROR" diff --git a/simplyblock_core/controllers/lvol_controller.py b/simplyblock_core/controllers/lvol_controller.py index 974cfab778..9be9b2084e 100644 --- a/simplyblock_core/controllers/lvol_controller.py +++ b/simplyblock_core/controllers/lvol_controller.py @@ -7,7 +7,7 @@ from datetime import datetime from typing import Any -from simplyblock_core import constants, index_ops, utils +from simplyblock_core import constants, floors, index_ops, utils from simplyblock_core.controllers import ( activation_gate, events_controller, @@ -1027,11 +1027,17 @@ def add_lvol_ha(name, size, host_id_or_name, ha_type, pool_id_or_name, use_comp= logger.error(mgs) return False, mgs - if ndcs or npcs: - if ndcs+npcs > len(online_nodes): - mgs = f"Online storage nodes: {len(online_nodes)} are less than the required LVol geometry: {(ndcs+npcs)}" - logger.error(mgs) - return False, mgs + # Resource floors (simplyblock_core.floors) for the volume's effective + # scheme -- its own ndcs/npcs when given, the cluster's otherwise -- + # judged on the ONLINE nodes, their data devices and (when used) failure + # domains. The refusal names the violated floor. + floor_violations = floors.evaluate(cl, snodes, ndcs=ndcs, npcs=npcs, online_nodes=online_nodes) + if floor_violations: + eff_ndcs, eff_npcs = floors.scheme(cl, ndcs, npcs) + mgs = (f"Cannot create the volume: the cluster is below the resource floors for the " + f"{eff_ndcs}+{eff_npcs} scheme: " + "; ".join(floor_violations)) + logger.error(mgs) + return False, mgs if cluster_size_total <= 0: # Every online device reported zero usable capacity: no meaningful diff --git a/simplyblock_core/floors.py b/simplyblock_core/floors.py new file mode 100644 index 0000000000..1917e86434 --- /dev/null +++ b/simplyblock_core/floors.py @@ -0,0 +1,251 @@ +"""Erasure-coding resource floors (product rules of 2026-11-09). + +A cluster with the scheme n+k (``cluster.distr_ndcs`` = n, ``cluster.distr_npcs`` += k) has five FLOORS. They bind: operator approval and ``cluster activate`` +refuse below a floor, and at runtime volume creation, planned node shutdown +and node removal refuse when the cluster AFTER the operation would be below +one. The status monitor reports DEGRADED while the online resources are below +a floor. An outage may take a running cluster below the floors; nothing is +suspended for that reason alone -- the data plane keeps serving and +migrating, and rebalancing restores the protection. + +1. data devices per node >= 1 +2. data devices (cluster-wide, online) >= n+k when the cluster declares + ``raid_devices`` (every data device is protected below simplyblock), else + >= n+2k +3. failure domains (when failure domains are used at all) >= n+k +4. nodes >= n+2k +5. journal devices are excluded from every device count + +Topology overrides: a single-node cluster needs 1 node and n+k data devices; +a two-node (dual / arbitration) cluster needs 2 nodes. + +Journal devices and the device model (floor 5) +---------------------------------------------- +A node's journal lives in ``StorageNode.jm_device`` (a ``JMDevice``), never in +``StorageNode.nvme_devices``: ``_prepare_cluster_devices_jm_on_dev`` moves the +device flagged ``NVMeDevice.STATUS_JM`` out of ``nvme_devices`` into +``jm_device`` and ``_prepare_cluster_devices_on_restart`` drops a STATUS_JM +entry from the list again. The three journal shapes therefore count like this: + +* partition journal nodes (``num_partitions_per_dev`` > 0, nvme and lblk + journal-split alike): every device in ``nvme_devices`` is a data device or + data partition; the journal partition(s) are only referenced from + ``jm_device`` and do not count; +* ``--enable-journal-device`` / ``--lblk-journal-device`` nodes: the whole + device that became the journal is ``jm_device`` and does not count; +* lblk journal-split nodes: the data partition is an ``nvme_devices`` entry + (``is_partition`` True) and counts; the journal partition is ``jm_device``. + +``count_data_devices`` nevertheless skips a device whose status is STATUS_JM +explicitly, for the window between flagging and preparation. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + +#: Device statuses that count as an online data device. The same set +#: ``_cluster_activate`` counts (READONLY and CANNOT_ALLOCATE devices still +#: hold their chunks and serve reads). +ONLINE_DEVICE_STATUSES = ( + NVMeDevice.STATUS_ONLINE, + NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE, +) + + +@dataclass(frozen=True) +class Floors: + """The binding minimums of one cluster (or one volume scheme on it).""" + + nodes: int + devices_per_node: int + data_devices: int + #: None when failure domains are not used on the cluster. + failure_domains: int | None + #: Human-readable formula behind each floor, for the violation text. + nodes_rule: str = "n+2k" + data_devices_rule: str = "n+2k" + failure_domains_rule: str = "n+k" + + +def _flag(obj, name: str) -> bool: + """A boolean model flag read defensively: a MagicMock stand-in or a record + that predates the field must read as False, never as truthy.""" + return getattr(obj, name, False) is True + + +def _int(value) -> int: + """An integer model field read defensively (see ``_flag``): a stand-in + that is not an int reads as 0, which no floor can be below.""" + return value if isinstance(value, int) and not isinstance(value, bool) else 0 + + +def scheme(cluster, ndcs: int | None = None, npcs: int | None = None) -> tuple[int, int]: + """``(n, k)``: the volume override when one is given, else the cluster's.""" + if ndcs or npcs: + return _int(ndcs), _int(npcs) + return _int(getattr(cluster, "distr_ndcs", 0)), _int(getattr(cluster, "distr_npcs", 0)) + + +def is_single_node_cluster(cluster, member_nodes: int | None) -> bool: + """Flagged single-node, activated as single-node, or one member.""" + return (_flag(cluster, "is_single_node") + or _flag(cluster, "activated_single_node") + or member_nodes == 1) + + +def is_two_node_cluster(cluster, member_nodes: int | None) -> bool: + """A dual-node cluster is one whose whole membership is two nodes + (``storage_node_ops.apply_jc_dual_node`` keys the JC dual-node flag on + exactly that); ``cluster.two_node_arbitration`` is the arbitrated variant + and only ever set on such a cluster.""" + return _flag(cluster, "two_node_arbitration") or member_nodes == 2 + + +def cluster_floors(cluster, uses_failure_domains: bool, member_nodes: int | None = None, + ndcs: int | None = None, npcs: int | None = None) -> Floors: + """The floors of ``cluster`` for the scheme n+k (the cluster's, or the + volume override ``ndcs``/``npcs``). + + ``member_nodes`` is the number of primary storage nodes that belong to + the cluster (every status but REMOVED): it decides the single-node and + two-node topology overrides together with the cluster flags. + ``uses_failure_domains`` is ``cluster.enable_failure_domain``. + """ + n, k = scheme(cluster, ndcs, npcs) + raid = _flag(cluster, "raid_devices") + + if raid: + data_devices, data_rule = n + k, "n+k, raid_devices" + else: + data_devices, data_rule = n + 2 * k, "n+2k" + nodes, nodes_rule = n + 2 * k, "n+2k" + + if is_single_node_cluster(cluster, member_nodes): + nodes, nodes_rule = 1, "single-node cluster" + # No redundancy to rebuild onto a spare: the stripe alone. + data_devices, data_rule = n + k, "n+k, single-node cluster" + elif is_two_node_cluster(cluster, member_nodes): + nodes, nodes_rule = 2, "two-node cluster" + + return Floors( + nodes=nodes, + devices_per_node=1, + data_devices=data_devices, + failure_domains=(n + k) if uses_failure_domains else None, + nodes_rule=nodes_rule, + data_devices_rule=data_rule, + ) + + +def is_data_device(dev) -> bool: + """An online data device: never the journal (floor 5, see the module + docstring), only the statuses that hold and serve chunks.""" + status = getattr(dev, "status", None) + if status == NVMeDevice.STATUS_JM: + return False + return status in ONLINE_DEVICE_STATUSES + + +def count_data_devices(nodes) -> tuple[int, dict[str, int]]: + """``(total, per_node)`` of online, non-journal data devices on ``nodes``. + + Every node given is counted (the caller decides which nodes are "online" + or "left after the operation"); a secondary-only node has no devices of + its own and is skipped. ``per_node`` has an entry for EVERY counted node, + zero included, so floor 1 (>= 1 data device per node) can be checked from + it. + """ + total = 0 + per_node: dict[str, int] = {} + for node in nodes: + if _flag(node, "is_secondary_node"): + continue + count = sum(1 for dev in (getattr(node, "nvme_devices", None) or []) + if is_data_device(dev)) + per_node[node.get_id()] = count + total += count + return total, per_node + + +def online_failure_domains(nodes) -> int: + """Distinct failure-domain ids (>= 0; -1 means untagged) among ``nodes``.""" + return len({node.failure_domain for node in nodes + if isinstance(getattr(node, "failure_domain", -1), int) + and node.failure_domain >= 0}) + + +def floor_violations(floors: Floors, online_nodes: int, data_devices: int, + per_node: dict[str, int], online_fds: int | None) -> list[str]: + """Every floor the given online counts violate, as human-readable text + naming the floor, e.g. ``"nodes online 3 < floor 4 (n+2k)"``. Empty when + every floor holds. ``online_fds`` None means failure domains are not + used (or not known) and floor 3 is not judged.""" + violations: list[str] = [] + if online_nodes < floors.nodes: + violations.append( + f"nodes online {online_nodes} < floor {floors.nodes} ({floors.nodes_rule})") + for node_id, count in sorted(per_node.items()): + if count < floors.devices_per_node: + violations.append( + f"node {node_id} has {count} online data devices < floor " + f"{floors.devices_per_node} (data devices per node)") + if data_devices < floors.data_devices: + violations.append( + f"data devices online {data_devices} < floor {floors.data_devices} " + f"({floors.data_devices_rule})") + if floors.failure_domains is not None and online_fds is not None \ + and online_fds < floors.failure_domains: + violations.append( + f"failure domains online {online_fds} < floor {floors.failure_domains} " + f"({floors.failure_domains_rule})") + return violations + + +def primary_members(snodes) -> list: + """The cluster's primary storage nodes that are members: every status but + REMOVED, secondary-only nodes left out.""" + return [n for n in snodes + if n.status != StorageNode.STATUS_REMOVED and not _flag(n, "is_secondary_node")] + + +def online_members(snodes) -> list: + return [n for n in primary_members(snodes) if n.status == StorageNode.STATUS_ONLINE] + + +def evaluate(cluster, snodes, exclude_node_id: str | None = None, + ndcs: int | None = None, npcs: int | None = None, + online_nodes=None) -> list[str]: + """The floor violations of ``cluster`` judged on the ONLINE resources of + ``snodes`` (every storage node of the cluster). + + ``exclude_node_id`` judges the cluster as it would be after that node + left (a planned shutdown or removal): the node is left out of the online + counts, but it stays a member for the topology overrides -- shutting one + node of a three-node cluster down does not make it a two-node cluster. + ``online_nodes`` replaces the ONLINE selection when the caller already + narrowed it (the monitor leaves nodes it does not count out). + """ + members = primary_members(snodes) + if online_nodes is None: + online_nodes = online_members(snodes) + online = [n for n in online_nodes + if n.get_id() != exclude_node_id and not _flag(n, "is_secondary_node")] + uses_fd = _flag(cluster, "enable_failure_domain") + floors = cluster_floors(cluster, uses_fd, member_nodes=len(members), ndcs=ndcs, npcs=npcs) + total, per_node = count_data_devices(online) + fds = online_failure_domains(online) if uses_fd else None + return floor_violations(floors, len(online), total, per_node, fds) + + +def migration_affinity_floor(cluster) -> str: + """The data plane's placement anti-affinity floor for a migration: + ``"relaxed"`` on a cluster whose data devices are protected below + simplyblock (``raid_devices``: several chunks of a stripe may share a + device when no other device is writable), ``"device"`` otherwise.""" + return "relaxed" if _flag(cluster, "raid_devices") else "device" diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index a8f6b88882..f62ff14283 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -181,6 +181,13 @@ def is_topology_owned(self) -> bool: node_label_suffix: str = "" disable_monitoring: bool = False strict_node_anti_affinity: bool = False + # Cluster-create-time, immutable declaration that every data device is + # protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers + # the data-device floor from n+2k to n+k (simplyblock_core.floors) and + # lets the data plane place several chunks of a stripe on one device when + # no other device is writable (allow_device_colocation on the distribs, + # affinity_floor "relaxed" on migrations). + raid_devices: bool = False tls: bool = False tls_config: dict = default_factory(dict) is_re_balancing: bool = False diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index 4ac45d9837..fc039e385d 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -9,12 +9,14 @@ cluster_ops, constants, db_controller, + floors, storage_network, storage_node_ops, utils, ) from simplyblock_core.controllers import ( activation_gate, + cluster_events, device_controller, health_controller, port_release_budget, @@ -632,9 +634,53 @@ def _count_status(cluster, snodes, jm_repl_by_node, dp_quorum_by_node, cluster_t online_nodes, online_devices, jm_replication_tasks) +# cluster id -> the floor violations last reported for it (the event and the +# warning are emitted when the set changes, not on every 3s tick). +_floor_violations_reported: dict = {} + + +def _apply_floors(cluster, snodes, status): + """DEGRADED, naming the floor, while the ONLINE resources are below a + floor (simplyblock_core.floors) and the existing rules did not already + SUSPEND the cluster. The suspension rules are untouched: an outage may + take a running cluster below the floors, nothing is suspended for that + alone -- the data plane keeps serving and migrating and the rebalance + restores the protection. The floor is recorded as a cluster event and a + warning whenever the set of violated floors changes.""" + violations = floors.evaluate(cluster, snodes) + key = cluster.get_id() + previous = _floor_violations_reported.get(key) + if violations != previous: + if violations: + logger.warning("Cluster %s below resource floor: %s", key, "; ".join(violations)) + try: + cluster_events.cluster_floor_violation(cluster, violations) + except Exception as e: # noqa: BLE001 - the status verdict must not fail on an event + logger.debug("floor event not recorded: %s", e) + elif previous: + logger.info("Cluster %s back at or above every resource floor", key) + try: + cluster_events.cluster_floor_restored(cluster) + except Exception as e: # noqa: BLE001 + logger.debug("floor event not recorded: %s", e) + _floor_violations_reported[key] = violations + if violations and status != Cluster.STATUS_SUSPENDED: + return Cluster.STATUS_DEGRADED + return status + + def _status_verdict(cluster, snodes, affected_physical_nodes, online_nodes, online_devices, jm_replication_tasks): - """The cluster status for one set of affected hosts.""" + """The cluster status for one set of affected hosts: the suspension / + degradation rules (``_status_verdict_rules``), then the floors.""" + status = _status_verdict_rules(cluster, snodes, affected_physical_nodes, online_nodes, + online_devices, jm_replication_tasks) + return _apply_floors(cluster, snodes, status) + + +def _status_verdict_rules(cluster, snodes, affected_physical_nodes, online_nodes, + online_devices, jm_replication_tasks): + """The suspension / degradation rules for one set of affected hosts.""" affected_nodes = len(affected_physical_nodes) # ndcs n = 2 # npcs k = 1 diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 174754e4cb..0876abcac8 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -36,6 +36,7 @@ cluster_ops, constants, distr_controller, + floors, jm_raid, scripts, storage_network, @@ -10278,29 +10279,6 @@ def _check_ftt_allows_node_removal(node_id, db_controller): if node.status not in [StorageNode.STATUS_IN_CREATION, StorageNode.STATUS_REMOVED] ) - # Block suspend/shutdown during rebalancing based on node headroom. - # A cluster needs ndcs+npcs nodes minimum. During rebalancing: - # - With exactly ndcs+npcs nodes: no shutdowns allowed (no headroom) - # - With ndcs+npcs+1 nodes: one shutdown allowed (one spare) - # - With ndcs+npcs+2+ nodes: two shutdowns allowed, etc. - # The number of allowed shutdowns during rebalancing is: - # total_active_nodes - (ndcs + npcs) - # This must be greater than the number of already-not-online nodes. - if cluster.is_re_balancing: - not_online_already = sum( - 1 for node in snodes - if node.get_id() != node_id - and node.status != StorageNode.STATUS_ONLINE - and node.status not in [StorageNode.STATUS_IN_CREATION, StorageNode.STATUS_REMOVED] - ) - headroom = total_active_nodes - (ndcs + npcs) - if headroom <= not_online_already: - return False, ( - f"Cluster is rebalancing with {total_active_nodes} active nodes " - f"({not_online_already} already not online, " - f"need >{ndcs + npcs} for ndcs={ndcs}, npcs={npcs}). " - f"Wait for rebalancing to complete before removing a node." - ) # Count nodes that are not online (excluding the node being removed, # and excluding nodes in creation or already removed). @@ -10476,6 +10454,21 @@ def _check_ftt_allows_node_removal(node_id, db_controller): f"(status: {not_online_node.status})" ) + # Resource floors (simplyblock_core.floors): the cluster AFTER losing + # this node -- its online nodes, their data devices and, when used, its + # failure domains -- must stay at or above every floor. This replaces + # the former "rebalancing headroom" formula (active nodes - (ndcs+npcs)): + # the node floor is n+2k whether or not a rebalance is running. Judged + # last: the FTT / pair / domain refusals above name the concrete node + # the operator has to wait for, which is the more actionable reason. + floor_violations = floors.evaluate(cluster, snodes, exclude_node_id=node_id) + if floor_violations: + return False, ( + f"removing node {node_id} would take the cluster below the resource floors " + f"for the {ndcs}+{npcs} scheme ({total_active_nodes} active nodes): " + + "; ".join(floor_violations) + + ". Bring the other nodes online or add resources first." + ) return True, "" @@ -10640,6 +10633,27 @@ def check_node_shutdown_preconditions(node_id, force=False, current_restart_task logger.error(reason) return False, reason + # Resource floors (simplyblock_core.floors): a planned shutdown is refused + # when the cluster without this node would be below a floor (nodes, data + # devices per node and cluster-wide, failure domains). force keeps + # overriding, as it does for every guard below. + if snode.status == StorageNode.STATUS_ONLINE: + try: + _floor_cluster = db_controller.get_cluster_by_id(snode.cluster_id) + except KeyError: + _floor_cluster = None + if _floor_cluster is not None: + _floor_violations = floors.evaluate( + _floor_cluster, db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id), + exclude_node_id=node_id) + if _floor_violations: + reason = (f"shutting down node {node_id} would take the cluster below the " + f"resource floors: " + "; ".join(_floor_violations)) + if force is False: + logger.error(reason) + return False, reason + logger.warning("%s — proceeding with force", reason) + # Guard: no concurrent shutdown + restart (design: mutual exclusion) for peer in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id): if peer.get_id() != node_id and peer.status == StorageNode.STATUS_RESTARTING: diff --git a/simplyblock_web/api/v1/cluster.py b/simplyblock_web/api/v1/cluster.py index 4ebcf265ae..dfa707d1f3 100644 --- a/simplyblock_web/api/v1/cluster.py +++ b/simplyblock_web/api/v1/cluster.py @@ -51,6 +51,7 @@ def add_cluster(): max_queue_size = cl_data.get('max_queue_size', 128) inflight_io_threshold = cl_data.get('inflight_io_threshold', 4) strict_node_anti_affinity = cl_data.get('strict_node_anti_affinity', False) + raid_devices = bool(cl_data.get('raid_devices', False)) is_single_node = cl_data.get('is_single_node', False) client_data_nic = cl_data.get('client_data_nic', "") max_fault_tolerance = min(distr_npcs, 2) if distr_npcs >= 1 else 1 @@ -66,6 +67,7 @@ def add_cluster(): cr_name, cr_namespace, cr_plural, fabric, client_data_nic=client_data_nic, max_fault_tolerance=max_fault_tolerance, backup_config=backup_config, nvmf_base_port=nvmf_base_port, rpc_base_port=rpc_base_port, snode_api_port=snode_api_port, + raid_devices=raid_devices, hashicorp_vault_settings=( HashicorpVaultSettings(raw_vault) if (raw_vault := cl_data.get('hashicorp_vault_settings')) is not None diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index 5215a97e2d..b72e16d0e1 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -210,6 +210,7 @@ class ClusterDTO(BaseModel): atomic_4k: bool page_copy: bool = False md_journal: bool = False + raid_devices: bool = False secret: SecretStr tls_enabled: bool max_fault_tolerance: int @@ -261,6 +262,7 @@ def from_model(model: Cluster, stat_obj: StatsObject | None = None): atomic_4k=model.atomic_4k, page_copy=model.page_copy, md_journal=model.md_journal, + raid_devices=model.raid_devices, secret=model.secret, tls_enabled=model.tls, max_fault_tolerance=model.max_fault_tolerance, diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index ca0aa6c616..1bb9845af6 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -85,6 +85,9 @@ class ClusterParams(BaseModel): inflight_io_threshold: int = 4 enable_node_affinity: bool = False strict_node_anti_affinity: bool = False + # Every data device is protected below simplyblock (hardware RAID LUN, md + # or dm RAID): data-device floor n+k instead of n+2k. Immutable. + raid_devices: bool = False is_single_node: bool = False fabric: str = "tcp" cr_name: str = "" diff --git a/tests/unit/test_activation_growth_guard.py b/tests/unit/test_activation_growth_guard.py index 33a3b88d1a..f57fc293d0 100644 --- a/tests/unit/test_activation_growth_guard.py +++ b/tests/unit/test_activation_growth_guard.py @@ -21,6 +21,10 @@ def _cluster(status=Cluster.STATUS_SUSPENDED, activated=()): cluster = MagicMock() cluster.status = status cluster.activated_node_ids = list(activated) + # Model defaults: as child mocks these read truthy and steer activation + # into the sync-replication / failure-domain branches. + cluster.sync_replication = False + cluster.enable_failure_domain = False cluster.get_id.return_value = "cl-1" return cluster @@ -37,9 +41,12 @@ def _nodes(*ids): class TestActivationRefusesAddedNodes(unittest.TestCase): def _activate(self, cluster, nodes): - """Run only far enough to hit (or pass) the growth guard.""" + """Run only far enough to hit (or pass) the growth guard. The resource + floors are not what these mock nodes model (test_cluster_floors covers + them), so they are patched out.""" with patch.object(cluster_ops, "db_controller") as db, \ patch.object(cluster_ops, "set_cluster_status"), \ + patch.object(cluster_ops.floors, "evaluate", return_value=[]), \ patch.object(cluster_ops, "logger"): db.get_cluster_by_id.return_value = cluster db.get_storage_nodes_by_cluster_id.return_value = nodes diff --git a/tests/unit/test_cluster_floors.py b/tests/unit/test_cluster_floors.py new file mode 100644 index 0000000000..e747a3442b --- /dev/null +++ b/tests/unit/test_cluster_floors.py @@ -0,0 +1,389 @@ +"""Unit tests for the erasure-coding resource floors (simplyblock_core.floors) +and the places that enforce them. + + - cluster_floors: the five floors for 2+1, 1+1, 2+2 and 4+2, with and + without raid_devices; the single-node and two-node topology overrides. + - count_data_devices: journal devices never count (floor 5). + - floor_violations: every violated floor is named. + - evaluate: online resources, the node left out of a planned operation. + - Activation: fresh activation refuses below a floor; the failure-domain + minimum is max(k+2, n+k). + - Monitor: DEGRADED (never SUSPENDED) while online resources are below a + floor, named in the cluster event. + - Shutdown preconditions: refused below a floor, force overrides. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import cluster_ops, floors, storage_node_ops +from simplyblock_core.controllers.cluster_expansion import planner +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import JMDevice, NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + + +def _cluster(ndcs, npcs, raid_devices=False, is_single_node=False, two_node_arbitration=False, + enable_failure_domain=False, ha_type="ha"): + cl = Cluster() + cl.uuid = "cluster-1" + cl.distr_ndcs = ndcs + cl.distr_npcs = npcs + cl.raid_devices = raid_devices + cl.is_single_node = is_single_node + cl.two_node_arbitration = two_node_arbitration + cl.enable_failure_domain = enable_failure_domain + cl.ha_type = ha_type + cl.max_fault_tolerance = max(npcs, 1) + cl.status = Cluster.STATUS_ACTIVE + return cl + + +def _dev(status=NVMeDevice.STATUS_ONLINE, uuid="dev"): + d = NVMeDevice() + d.uuid = uuid + d.status = status + return d + + +def _node(uuid, devices=1, status=StorageNode.STATUS_ONLINE, failure_domain=-1, + device_statuses=None, with_jm=False): + n = StorageNode() + n.uuid = uuid + n.cluster_id = "cluster-1" + n.status = status + n.mgmt_ip = f"10.0.0.{abs(hash(uuid)) % 250 + 1}" + n.failure_domain = failure_domain + statuses = device_statuses if device_statuses is not None else [NVMeDevice.STATUS_ONLINE] * devices + n.nvme_devices = [_dev(s, f"{uuid}-dev-{i}") for i, s in enumerate(statuses)] + if with_jm: + jm = JMDevice() + jm.uuid = f"{uuid}-jm" + jm.status = NVMeDevice.STATUS_ONLINE + n.jm_device = jm + return n + + +# --------------------------------------------------------------------------- +# cluster_floors +# --------------------------------------------------------------------------- + +class TestClusterFloors(unittest.TestCase): + + def _floors(self, ndcs, npcs, members=8, **kw): + cl = _cluster(ndcs, npcs, **kw) + return floors.cluster_floors(cl, uses_failure_domains=kw.get("enable_failure_domain", False), + member_nodes=members) + + def test_2_1(self): + f = self._floors(2, 1) + self.assertEqual((f.nodes, f.devices_per_node, f.data_devices, f.failure_domains), (4, 1, 4, None)) + f = self._floors(2, 1, raid_devices=True) + self.assertEqual((f.nodes, f.data_devices), (4, 3)) + self.assertIn("raid_devices", f.data_devices_rule) + + def test_1_1(self): + f = self._floors(1, 1) + self.assertEqual((f.nodes, f.data_devices), (3, 3)) + self.assertEqual(self._floors(1, 1, raid_devices=True).data_devices, 2) + + def test_2_2(self): + f = self._floors(2, 2) + self.assertEqual((f.nodes, f.data_devices), (6, 6)) + self.assertEqual(self._floors(2, 2, raid_devices=True).data_devices, 4) + + def test_4_2(self): + f = self._floors(4, 2) + self.assertEqual((f.nodes, f.data_devices), (8, 8)) + self.assertEqual(self._floors(4, 2, raid_devices=True).data_devices, 6) + + def test_failure_domain_floor_is_n_plus_k_only_when_used(self): + self.assertEqual(self._floors(4, 2, enable_failure_domain=True).failure_domains, 6) + self.assertEqual(self._floors(2, 1, enable_failure_domain=True).failure_domains, 3) + self.assertIsNone(self._floors(2, 1).failure_domains) + + def test_single_node_override(self): + for kw in ({"is_single_node": True, "members": 3}, {"members": 1}): + f = self._floors(2, 1, **kw) + self.assertEqual((f.nodes, f.data_devices), (1, 3), kw) + self.assertEqual(f.nodes_rule, "single-node cluster") + cl = _cluster(1, 0) + cl.activated_single_node = True + f = floors.cluster_floors(cl, False, member_nodes=1) + self.assertEqual((f.nodes, f.data_devices), (1, 1)) + + def test_two_node_override(self): + for kw in ({"two_node_arbitration": True, "members": 2}, {"members": 2}): + f = self._floors(1, 1, **kw) + # Two nodes, but the data-device floor stays the cluster rule. + self.assertEqual((f.nodes, f.data_devices), (2, 3), kw) + self.assertEqual(f.nodes_rule, "two-node cluster") + self.assertEqual(self._floors(1, 1, raid_devices=True, members=2).data_devices, 2) + + def test_volume_scheme_override(self): + cl = _cluster(4, 2) + f = floors.cluster_floors(cl, False, member_nodes=8, ndcs=2, npcs=1) + self.assertEqual((f.nodes, f.data_devices), (4, 4)) + + def test_mock_flags_never_read_truthy(self): + """A MagicMock stand-in for the cluster (spec'd or not) must not turn a + child mock into raid_devices / two_node / single-node.""" + cl = MagicMock(spec=Cluster) + cl.distr_ndcs, cl.distr_npcs = 2, 1 + f = floors.cluster_floors(cl, False, member_nodes=5) + self.assertEqual((f.nodes, f.data_devices), (4, 4)) + + +# --------------------------------------------------------------------------- +# count_data_devices (floor 5: journal devices never count) +# --------------------------------------------------------------------------- + +class TestCountDataDevices(unittest.TestCase): + + def test_journal_is_not_a_data_device(self): + # A dedicated journal device lives in jm_device, outside nvme_devices; + # a device still flagged STATUS_JM inside the list is skipped as well. + n1 = _node("n1", device_statuses=[NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_JM], with_jm=True) + n2 = _node("n2", devices=2, with_jm=True) + total, per_node = floors.count_data_devices([n1, n2]) + self.assertEqual(total, 3) + self.assertEqual(per_node, {"n1": 1, "n2": 2}) + + def test_only_serving_statuses_count(self): + n = _node("n1", device_statuses=[ + NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE, + NVMeDevice.STATUS_UNAVAILABLE, NVMeDevice.STATUS_NEW, NVMeDevice.STATUS_FAILED, + NVMeDevice.STATUS_REMOVED, NVMeDevice.STATUS_FAILED_AND_MIGRATED]) + total, per_node = floors.count_data_devices([n]) + self.assertEqual((total, per_node["n1"]), (3, 3)) + + def test_a_node_without_devices_is_listed_with_zero(self): + total, per_node = floors.count_data_devices([_node("n1", devices=0), _node("n2")]) + self.assertEqual((total, per_node), (1, {"n1": 0, "n2": 1})) + + def test_secondary_only_nodes_are_skipped(self): + sec = _node("sec", devices=0) + sec.is_secondary_node = True + total, per_node = floors.count_data_devices([sec, _node("n1")]) + self.assertEqual((total, per_node), (1, {"n1": 1})) + + +# --------------------------------------------------------------------------- +# floor_violations / evaluate +# --------------------------------------------------------------------------- + +class TestViolations(unittest.TestCase): + + def test_every_violated_floor_is_named(self): + f = floors.Floors(nodes=4, devices_per_node=1, data_devices=4, failure_domains=3) + v = floors.floor_violations(f, online_nodes=3, data_devices=2, per_node={"a": 2, "b": 0}, online_fds=2) + self.assertEqual(v, [ + "nodes online 3 < floor 4 (n+2k)", + "node b has 0 online data devices < floor 1 (data devices per node)", + "data devices online 2 < floor 4 (n+2k)", + "failure domains online 2 < floor 3 (n+k)", + ]) + + def test_no_violation_is_empty(self): + f = floors.Floors(nodes=4, devices_per_node=1, data_devices=4, failure_domains=None) + self.assertEqual(floors.floor_violations(f, 4, 4, {"a": 1, "b": 1, "c": 1, "d": 1}, None), []) + + def test_evaluate_judges_online_resources(self): + cl = _cluster(2, 1) + nodes = [_node(f"n{i}") for i in range(4)] + [_node("n4", status=StorageNode.STATUS_OFFLINE)] + self.assertEqual(floors.evaluate(cl, nodes), []) + nodes[3].status = StorageNode.STATUS_OFFLINE + self.assertEqual(floors.evaluate(cl, nodes), [ + "nodes online 3 < floor 4 (n+2k)", + "data devices online 3 < floor 4 (n+2k)", + ]) + + def test_evaluate_without_a_node_keeps_the_membership(self): + """Shutting one node of three down does not turn a 1+1 cluster into a + two-node cluster: the excluded node still counts as a member.""" + cl = _cluster(1, 1) + nodes = [_node("n0", devices=2), _node("n1", devices=2), _node("n2", devices=2)] + self.assertEqual(floors.evaluate(cl, nodes), []) + self.assertEqual(floors.evaluate(cl, nodes, exclude_node_id="n2"), + ["nodes online 2 < floor 3 (n+2k)"]) + + def test_evaluate_failure_domains(self): + cl = _cluster(2, 1, enable_failure_domain=True) + nodes = [_node(f"n{i}", failure_domain=i % 3) for i in range(6)] + self.assertEqual(floors.evaluate(cl, nodes), []) + for n in nodes: + n.failure_domain = n.failure_domain % 2 + self.assertEqual(floors.evaluate(cl, nodes), ["failure domains online 2 < floor 3 (n+k)"]) + + def test_evaluate_volume_override(self): + cl = _cluster(1, 1) + nodes = [_node(f"n{i}") for i in range(3)] + self.assertEqual(floors.evaluate(cl, nodes), []) + self.assertEqual(floors.evaluate(cl, nodes, ndcs=2, npcs=1), + ["nodes online 3 < floor 4 (n+2k)", "data devices online 3 < floor 4 (n+2k)"]) + + def test_migration_affinity_floor(self): + self.assertEqual(floors.migration_affinity_floor(_cluster(2, 1)), "device") + self.assertEqual(floors.migration_affinity_floor(_cluster(2, 1, raid_devices=True)), "relaxed") + + +# --------------------------------------------------------------------------- +# Activation +# --------------------------------------------------------------------------- + +class TestActivation(unittest.TestCase): + + def test_fresh_activation_floor_check(self): + cl = _cluster(2, 1) + nodes = [_node(f"n{i}") for i in range(4)] + self.assertEqual(cluster_ops.activation_floor_violations(cl, nodes, nodes), []) + nodes.append(_node("n4", devices=0)) + self.assertEqual(cluster_ops.activation_floor_violations(cl, nodes, nodes), + ["node n4 has 0 online data devices < floor 1 (data devices per node)"]) + + def test_single_node_activation_floor(self): + cl = _cluster(1, 0, is_single_node=True) + nodes = [_node("n0")] + self.assertEqual(cluster_ops.activation_floor_violations(cl, nodes, nodes), []) + cl = _cluster(2, 1, is_single_node=True) + nodes = [_node("n0", devices=3)] + self.assertEqual(cluster_ops.activation_floor_violations(cl, nodes, nodes), []) + nodes = [_node("n0", devices=2)] + self.assertEqual(cluster_ops.activation_floor_violations(cl, nodes, nodes), + ["data devices online 2 < floor 3 (n+k, single-node cluster)"]) + + def test_reactivation_gate_is_the_old_device_count(self): + self.assertEqual(cluster_ops.activation_minimum_devices(_cluster(2, 1), False), 4) + self.assertEqual(cluster_ops.activation_minimum_devices(_cluster(2, 1), True), 3) + + def test_fd_activation_minimum_is_max_of_spare_rule_and_floor(self): + self.assertEqual(planner.fd_activation_min_domains(1, 2), 3) # 2+1: k+2 == n+k + self.assertEqual(planner.fd_activation_min_domains(2, 4), 6) # 4+2: n+k binds + self.assertEqual(planner.fd_activation_min_domains(1, 1), 3) # 1+1: k+2 binds + self.assertIsNone(planner.fd_activation_domain_count_violation(2, 6, ndcs=4)) + msg = planner.fd_activation_domain_count_violation(2, 5, ndcs=4) + self.assertIn("at least 6 distinct failure domains (n+k", msg) + # Without ndcs the former k+2 rule is unchanged. + self.assertIsNone(planner.fd_activation_domain_count_violation(1, 3)) + self.assertIsNotNone(planner.fd_activation_domain_count_violation(1, 2)) + + +# --------------------------------------------------------------------------- +# Monitor +# --------------------------------------------------------------------------- + +class TestMonitorFloors(unittest.TestCase): + + def _status(self, cluster, nodes): + from simplyblock_core.services import storage_node_monitor as mod + mod._floor_violations_reported.clear() + events = MagicMock() + with patch.object(mod, "db") as mock_db, patch.object(mod, "cluster_events", events), \ + patch.object(mod, "is_new_migrated_node", return_value=False): + mock_db.get_cluster_by_id.return_value = cluster + mock_db.get_primary_storage_nodes_by_cluster_id.return_value = nodes + mock_db.get_job_tasks.return_value = [] + return mod.get_next_cluster_status("cluster-1"), events + + def test_healthy_cluster_at_the_floor_is_active(self): + status, events = self._status(_cluster(2, 1), [_node(f"n{i}") for i in range(4)]) + self.assertEqual(status, Cluster.STATUS_ACTIVE) + events.cluster_floor_violation.assert_not_called() + + def test_below_the_node_floor_is_degraded_and_named(self): + status, events = self._status(_cluster(2, 1), [_node(f"n{i}", devices=2) for i in range(3)]) + self.assertEqual(status, Cluster.STATUS_DEGRADED) + _, violations = events.cluster_floor_violation.call_args.args + self.assertEqual(violations, ["nodes online 3 < floor 4 (n+2k)"]) + + def test_below_the_device_floor_is_degraded_not_suspended(self): + # Every node online, the data devices below n+2k: the old rules say + # ACTIVE (n+k devices are there), the floors say DEGRADED. + nodes = [_node(f"n{i}") for i in range(4)] + nodes[3].nvme_devices = [] + status, _ = self._status(_cluster(2, 1), nodes) + self.assertEqual(status, Cluster.STATUS_DEGRADED) + + def test_raid_devices_lowers_the_device_floor_for_the_monitor(self): + nodes = [_node(f"n{i}") for i in range(4)] + nodes[3].nvme_devices = [_dev(NVMeDevice.STATUS_ONLINE)] + nodes[2].nvme_devices = [] + cl = _cluster(2, 1, raid_devices=True) + status, events = self._status(cl, nodes) + # The per-node floor still degrades; the cluster-wide one no longer does. + self.assertEqual(status, Cluster.STATUS_DEGRADED) + _, violations = events.cluster_floor_violation.call_args.args + self.assertEqual(violations, ["node n2 has 0 online data devices < floor 1 (data devices per node)"]) + + def test_suspension_rules_are_untouched(self): + # k=1 with two affected nodes: SUSPENDED by the existing rules, and + # the floors never turn that into DEGRADED. + nodes = [_node(f"n{i}") for i in range(5)] + nodes[0].status = StorageNode.STATUS_OFFLINE + nodes[1].status = StorageNode.STATUS_OFFLINE + status, _ = self._status(_cluster(2, 1), nodes) + self.assertEqual(status, Cluster.STATUS_SUSPENDED) + + def test_event_once_per_change_and_restored(self): + from simplyblock_core.services import storage_node_monitor as mod + cl = _cluster(2, 1) + short = [_node(f"n{i}", devices=2) for i in range(3)] + whole = short + [_node("n3", devices=2)] + mod._floor_violations_reported.clear() + events = MagicMock() + with patch.object(mod, "db") as mock_db, patch.object(mod, "cluster_events", events), \ + patch.object(mod, "is_new_migrated_node", return_value=False): + mock_db.get_cluster_by_id.return_value = cl + mock_db.get_job_tasks.return_value = [] + mock_db.get_primary_storage_nodes_by_cluster_id.return_value = short + mod.get_next_cluster_status("cluster-1") + mod.get_next_cluster_status("cluster-1") + self.assertEqual(events.cluster_floor_violation.call_count, 1) + mock_db.get_primary_storage_nodes_by_cluster_id.return_value = whole + self.assertEqual(mod.get_next_cluster_status("cluster-1"), Cluster.STATUS_ACTIVE) + events.cluster_floor_restored.assert_called_once() + + +# --------------------------------------------------------------------------- +# Shutdown preconditions +# --------------------------------------------------------------------------- + +class TestShutdownFloors(unittest.TestCase): + + def _db(self, cluster, nodes): + db = MagicMock() + db.get_cluster_by_id.return_value = cluster + db.get_storage_nodes_by_cluster_id.return_value = nodes + db.get_storage_node_by_id.side_effect = lambda nid: next(n for n in nodes if n.get_id() == nid) + return db + + def test_shutdown_below_a_floor_is_refused(self): + nodes = [_node(f"n{i}", devices=2) for i in range(4)] + with patch.object(storage_node_ops, "DBController", return_value=self._db(_cluster(2, 1), nodes)): + allowed, reason = storage_node_ops.check_node_shutdown_preconditions("n0") + self.assertFalse(allowed) + self.assertIn("nodes online 3 < floor 4 (n+2k)", reason) + + def test_force_overrides_the_floor(self): + nodes = [_node(f"n{i}", devices=2) for i in range(4)] + tc = MagicMock() + tc.get_active_node_tasks.return_value = [] + tc.get_active_node_restart_task.return_value = [] + with patch.object(storage_node_ops, "DBController", return_value=self._db(_cluster(2, 1), nodes)), \ + patch.object(storage_node_ops, "tasks_controller", tc): + allowed, reason = storage_node_ops.check_node_shutdown_preconditions("n0", force=True) + self.assertTrue(allowed, reason) + + def test_shutdown_with_headroom_passes_the_floor(self): + nodes = [_node(f"n{i}", devices=2) for i in range(5)] + tc = MagicMock() + tc.get_active_node_tasks.return_value = [] + tc.get_active_node_restart_task.return_value = [] + with patch.object(storage_node_ops, "DBController", return_value=self._db(_cluster(2, 1), nodes)), \ + patch.object(storage_node_ops, "tasks_controller", tc): + allowed, reason = storage_node_ops.check_node_shutdown_preconditions("n0") + self.assertTrue(allowed, reason) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_failure_domain.py b/tests/unit/test_failure_domain.py index ac7f3c6560..88034ab870 100644 --- a/tests/unit/test_failure_domain.py +++ b/tests/unit/test_failure_domain.py @@ -616,6 +616,10 @@ def setUp(self): self.snm = snm self._patches = [ patch.object(snm, "is_new_migrated_node", return_value=False), + # These 2-domain layouts test the FD suspend contract; they sit + # below the n+k failure-domain floor, which adds DEGRADED on its + # own (covered in test_cluster_floors). + patch.object(snm.floors, "evaluate", return_value=[]), ] for p in self._patches: p.start() diff --git a/tests/unit/test_ftt_protection.py b/tests/unit/test_ftt_protection.py index ddbdd07086..17b76202da 100644 --- a/tests/unit/test_ftt_protection.py +++ b/tests/unit/test_ftt_protection.py @@ -5,7 +5,10 @@ - FTT=1 (npcs=1): block if any node not online or journal replicating - FTT=2 (npcs=2, ft=2): allow one not-online, block at two - npcs=2, ft=1: like FTT=2 plus primary-secondary pair constraint - - Rebalancing: block based on headroom (active_nodes - ndcs - npcs) + - Resource floors (simplyblock_core.floors) are judged last: the removal is + refused when the cluster without the node would be below a floor. The + FTT tests below patch the floors out (their fixtures model the FTT rule + with as few nodes as it takes); TestFloorGate exercises the gate itself. - Journal replication counts as one additional not-online node - REMOVED and IN_CREATION nodes are excluded from counts """ @@ -13,7 +16,11 @@ import unittest from unittest.mock import MagicMock, patch +import pytest + +from simplyblock_core import storage_node_ops from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.rpc_client import RPCConnectionError from simplyblock_core.storage_node_ops import _check_ftt_allows_node_removal @@ -78,6 +85,30 @@ def _set_jm_replicating(node): rpc.jc_get_jm_status.return_value = {"jm1": False} +def _with_devices(node, count=1): + """Give a mock node ``count`` online data devices (the floor gate counts them).""" + devs = [] + for i in range(count): + d = MagicMock(spec=NVMeDevice) + d.status = NVMeDevice.STATUS_ONLINE + d.get_id = MagicMock(return_value=f"{node.get_id()}-dev-{i}") + devs.append(d) + node.nvme_devices = devs + return node + + +@pytest.fixture(autouse=True) +def _floors_out_of_the_ftt_tests(request): + """The FTT fixtures are sized for the FTT rule alone; the resource floors + (n+2k nodes, one data device per node, ...) are a separate gate with its + own tests (TestFloorGate), so they are patched out everywhere else.""" + if request.cls is not None and request.cls.__name__ == "TestFloorGate": + yield + return + with patch.object(storage_node_ops.floors, "evaluate", return_value=[]): + yield + + # --------------------------------------------------------------------------- # Non-HA clusters # --------------------------------------------------------------------------- @@ -278,101 +309,6 @@ def test_two_unrelated_offline_blocks(self): allowed, reason = _check_ftt_allows_node_removal("n1", db) self.assertFalse(allowed) self.assertIn("npcs=2/ft=1", reason) - - -# --------------------------------------------------------------------------- -# Rebalancing -# --------------------------------------------------------------------------- - -class TestRebalancing(unittest.TestCase): - - def test_rebalancing_exact_minimum_nodes_blocks(self): - """ndcs=2, npcs=1: minimum=3. With exactly 3 nodes and rebalancing → block.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3")] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_rebalancing_one_above_minimum_allows(self): - """ndcs=2, npcs=1: minimum=3. With 4 nodes and rebalancing → allowed.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3"), _node("n4")] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n1", db) - self.assertTrue(allowed) - - def test_rebalancing_one_above_min_but_one_offline_blocks(self): - """ndcs=2, npcs=1: 4 nodes, 1 offline, rebalancing → headroom=1, not_online=1 → block.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3"), - _node("n4", status=StorageNode.STATUS_OFFLINE)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_rebalancing_two_above_min_one_offline_allows_ftt2(self): - """ndcs=2, npcs=2, ft=2: 6 nodes, 1 offline, rebalancing → headroom=2, not_online=1 → allowed.""" - cl = _cluster(npcs=2, ndcs=2, ft=2, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3"), _node("n4"), - _node("n5"), _node("n6", status=StorageNode.STATUS_OFFLINE)] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n1", db) - self.assertTrue(allowed) - - def test_rebalancing_npcs2_exact_minimum_blocks(self): - """ndcs=2, npcs=2: minimum=4. With exactly 4 nodes and rebalancing → block.""" - cl = _cluster(npcs=2, ndcs=2, ft=2, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3"), _node("n4")] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_rebalancing_npcs2_one_above_allows_first(self): - """ndcs=2, npcs=2: 5 nodes, rebalancing → headroom=1, not_online=0 → allowed.""" - cl = _cluster(npcs=2, ndcs=2, ft=2, rebalancing=True) - nodes = [_node(f"n{i}") for i in range(5)] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n0", db) - self.assertTrue(allowed) - - def test_rebalancing_npcs2_one_above_blocks_second(self): - """ndcs=2, npcs=2: 5 nodes, 1 offline, rebalancing → headroom=1, not_online=1 → block.""" - cl = _cluster(npcs=2, ndcs=2, ft=2, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3"), _node("n4"), - _node("n5", status=StorageNode.STATUS_OFFLINE)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_not_rebalancing_exact_minimum_allows(self): - """Same scenario but NOT rebalancing → rebalancing check doesn't apply.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=False) - nodes = [_node("n1"), _node("n2"), _node("n3")] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n1", db) - self.assertTrue(allowed) - - def test_rebalancing_removed_nodes_excluded(self): - """Removed nodes don't count as active, reducing headroom.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - # 4 nodes but one is removed → 3 active → exact minimum → block - nodes = [_node("n1"), _node("n2"), _node("n3"), - _node("n4", status=StorageNode.STATUS_REMOVED)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - -# --------------------------------------------------------------------------- -# Edge cases -# --------------------------------------------------------------------------- - class TestEdgeCases(unittest.TestCase): def test_node_not_found(self): @@ -849,73 +785,6 @@ def test_fd_disabled_ignores_domains_uses_node_count(self): allowed, reason = _check_ftt_allows_node_removal("n1", db) self.assertFalse(allowed) self.assertIn("FTT=1", reason) - - -# --------------------------------------------------------------------------- -# Rebalancing additional scenarios -# --------------------------------------------------------------------------- - -class TestRebalancingAdditional(unittest.TestCase): - - def test_rebalancing_ndcs4_npcs2_six_nodes_blocks(self): - """ndcs=4, npcs=2: minimum=6. With 6 nodes, rebalancing → headroom=0 → block.""" - cl = _cluster(npcs=2, ndcs=4, ft=2, rebalancing=True) - nodes = [_node(f"n{i}") for i in range(6)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n0", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_rebalancing_ndcs4_npcs2_seven_nodes_allows(self): - """ndcs=4, npcs=2: minimum=6. With 7 nodes, rebalancing → headroom=1 → allowed.""" - cl = _cluster(npcs=2, ndcs=4, ft=2, rebalancing=True) - nodes = [_node(f"n{i}") for i in range(7)] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n0", db) - self.assertTrue(allowed) - - def test_rebalancing_ndcs1_npcs1_two_nodes_blocks(self): - """ndcs=1, npcs=1: minimum=2. With 2 nodes, rebalancing → block.""" - cl = _cluster(npcs=1, ndcs=1, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2")] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - def test_rebalancing_ndcs1_npcs1_three_nodes_allows(self): - """ndcs=1, npcs=1: minimum=2. With 3 nodes, rebalancing → allowed.""" - cl = _cluster(npcs=1, ndcs=1, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3")] - db = _db(cl, nodes) - allowed, _ = _check_ftt_allows_node_removal("n1", db) - self.assertTrue(allowed) - - def test_rebalancing_in_creation_not_counted_as_active(self): - """in_creation nodes reduce active count, reducing headroom.""" - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - # 4 nodes but 1 in_creation → 3 active → minimum → block - nodes = [_node("n1"), _node("n2"), _node("n3"), - _node("n4", status=StorageNode.STATUS_IN_CREATION)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - - def test_rebalancing_check_happens_before_ftt_check(self): - """Even if FTT would allow it, rebalancing with no headroom blocks.""" - cl = _cluster(npcs=2, ndcs=2, ft=2, rebalancing=True) - # 4 nodes, all online, FTT=2 would allow but rebalancing blocks (headroom=0) - nodes = [_node(f"n{i}") for i in range(4)] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n0", db) - self.assertFalse(allowed) - self.assertIn("rebalancing", reason) - - -# --------------------------------------------------------------------------- -# Reason string validation -# --------------------------------------------------------------------------- - class TestReasonStrings(unittest.TestCase): def test_ftt1_reason_includes_count(self): @@ -939,15 +808,6 @@ def test_ftt2_reason_includes_count(self): self.assertFalse(allowed) self.assertIn("2 not-online", reason) - def test_rebalancing_reason_includes_node_counts(self): - cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) - nodes = [_node("n1"), _node("n2"), _node("n3")] - db = _db(cl, nodes) - allowed, reason = _check_ftt_allows_node_removal("n1", db) - self.assertFalse(allowed) - self.assertIn("3 active nodes", reason) - self.assertIn("rebalancing", reason) - def test_pair_reason_includes_node_ids(self): cl = _cluster(npcs=2, ft=1) nodes = [_node("n1", secondary_id="n2"), @@ -1099,3 +959,85 @@ def test_tertiary_counts_as_own_secondary(self): if __name__ == "__main__": unittest.main() + + +# --------------------------------------------------------------------------- +# Resource floors: the cluster after the removal must stay at or above them +# --------------------------------------------------------------------------- + +class TestFloorGate(unittest.TestCase): + """2+1: node floor n+2k = 4, data-device floor 4 (n+k = 3 with + raid_devices), one data device per node.""" + + def _nodes(self, count, devices=1): + return [_with_devices(_node(f"n{i}"), devices) for i in range(count)] + + def test_removal_that_leaves_the_node_floor_is_allowed(self): + cl = _cluster(npcs=1, ndcs=2, ft=1) + db = _db(cl, self._nodes(5, devices=2)) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed, reason) + + def test_removal_below_the_node_floor_is_refused_and_named(self): + cl = _cluster(npcs=1, ndcs=2, ft=1) + db = _db(cl, self._nodes(4, devices=2)) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + self.assertIn("nodes online 3 < floor 4 (n+2k)", reason) + + def test_removal_below_the_data_device_floor_is_refused(self): + cl = _cluster(npcs=1, ndcs=2, ft=1) + nodes = self._nodes(5, devices=1) + # Without n1 four devices remain -- the floor. Take one away. + nodes[4].nvme_devices = [] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + self.assertIn("node n4 has 0 online data devices < floor 1", reason) + self.assertIn("data devices online 3 < floor 4 (n+2k)", reason) + + def test_raid_devices_lowers_the_data_device_floor(self): + cl = _cluster(npcs=1, ndcs=2, ft=1) + cl.raid_devices = True + nodes = self._nodes(5, devices=1) + nodes[4].nvme_devices = [] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + # The per-node floor still refuses; the cluster-wide one no longer does. + self.assertFalse(allowed) + self.assertIn("node n4 has 0 online data devices", reason) + self.assertNotIn("data devices online", reason) + + def test_journal_device_does_not_count(self): + cl = _cluster(npcs=1, ndcs=2, ft=1) + nodes = self._nodes(5, devices=1) + jm = MagicMock(spec=NVMeDevice) + jm.status = NVMeDevice.STATUS_JM + jm.get_id = MagicMock(return_value="n4-jm") + nodes[4].nvme_devices = [jm] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertFalse(allowed) + self.assertIn("node n4 has 0 online data devices", reason) + + def test_rebalancing_no_longer_adds_a_headroom_rule(self): + """The former headroom formula (active nodes - (ndcs+npcs)) is gone: + with the floors satisfied a rebalance does not block the removal.""" + cl = _cluster(npcs=1, ndcs=2, ft=1, rebalancing=True) + db = _db(cl, self._nodes(5, devices=2)) + allowed, reason = _check_ftt_allows_node_removal("n1", db) + self.assertTrue(allowed, reason) + + def test_failure_domain_floor(self): + cl = _cluster(npcs=1, ndcs=2, ft=1, enable_failure_domain=True) + # Six nodes in three domains: removing n0 leaves domains {0,1,2} = n+k. + nodes = [_with_devices(_node(f"n{i}", failure_domain=i % 3), 1) for i in range(6)] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n0", db) + self.assertTrue(allowed, reason) + # Two domains only: below the n+k = 3 domain floor. + nodes = [_with_devices(_node(f"n{i}", failure_domain=i % 2), 1) for i in range(6)] + db = _db(cl, nodes) + allowed, reason = _check_ftt_allows_node_removal("n0", db) + self.assertFalse(allowed) + self.assertIn("failure domains online 2 < floor 3 (n+k)", reason) diff --git a/tests/unit/test_node_removal.py b/tests/unit/test_node_removal.py index e3283bb557..2e38c5c41a 100644 --- a/tests/unit/test_node_removal.py +++ b/tests/unit/test_node_removal.py @@ -19,6 +19,8 @@ import unittest from unittest.mock import DEFAULT, MagicMock, call, patch +import pytest + from simplyblock_core import storage_node_ops from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.nvme_device import JMDevice, NVMeDevice, RemoteJMDevice @@ -26,6 +28,16 @@ from simplyblock_core.rpc_client import RPCConnectionError, RPCException, RPCRemoteError from tests._mocks import unique_ip + +@pytest.fixture(autouse=True) +def _floors_out_of_the_removal_tests(): + """These fixtures (one to four nodes, no devices) model the removal + mechanics, not the resource floors (simplyblock_core.floors), which would + refuse every removal here; the floors have their own tests + (test_ftt_protection.TestFloorGate, test_cluster_floors).""" + with patch.object(storage_node_ops.floors, "evaluate", return_value=[]): + yield + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- diff --git a/tests/unit/test_removal_three_steps.py b/tests/unit/test_removal_three_steps.py index b2e2f50343..284a126427 100644 --- a/tests/unit/test_removal_three_steps.py +++ b/tests/unit/test_removal_three_steps.py @@ -83,7 +83,9 @@ def test_k1_is_degraded_only_by_the_removal_while_its_devices_rebuild(self): def test_k2_stays_active_while_the_removal_rebuilds(self): leaving = _node("x", "10.0.9.9", StorageNode.STATUS_MIGRATING_DEVICES, (NVMeDevice.STATUS_FAILED,)) - status, _ = self._verdicts(_cluster(npcs=2), self._online(5) + [leaving]) + # Six online nodes: the 2+2 node floor (n+2k), so the floors do not + # degrade the verdict on their own. + status, _ = self._verdicts(_cluster(npcs=2), self._online(6) + [leaving]) assert status == Cluster.STATUS_ACTIVE def test_k1_the_removals_own_shutdown_is_already_the_removals(self): diff --git a/tests/unit/test_sync_replication_site_loss.py b/tests/unit/test_sync_replication_site_loss.py index 3d76a422db..614b6431d4 100644 --- a/tests/unit/test_sync_replication_site_loss.py +++ b/tests/unit/test_sync_replication_site_loss.py @@ -17,6 +17,15 @@ from simplyblock_core.services import tasks_runner_sync_promote as runner from simplyblock_core.snode_client import SNodeClientException + +@pytest.fixture(autouse=True) +def _floors_out_of_the_site_tests(): + """The site verdicts are judged on the veto / majority rules alone; the + resource floors (simplyblock_core.floors, DEGRADED while a site's online + nodes are below n+2k) are tested in test_cluster_floors.""" + with patch.object(storage_node_monitor.floors, "evaluate", return_value=[]): + yield + API_UP = "the node API reports the SPDK process up" RPC_ANSWERS = "its RPC answers" From 06221031f5620f4940316f91fbfe1037e4e90311 Mon Sep 17 00:00:00 2001 From: michael Date: Mon, 9 Nov 2026 14:58:30 +0100 Subject: [PATCH 04/50] feat(migration): relax at exactly n+k nodes, plumb the affinity floor - needs_relaxed_placement: `<=` n+k member nodes (exactly n+k has no strict target for a moved chunk; closes a migration dead end). - migration_affinity_floor(cluster): "relaxed" with raid_devices, else "device"; passed by every migration runner as affinity_floor. - rpc_client: optional affinity_floor on distr_migration_failure_start / distr_migration_expansion_start and allow_device_colocation on bdev_distrib_create, sent only while constants.DISTR_AFFINITY_FLOOR_RPC is True (False until the ultra feat/affinity-floor data plane ships; older data planes reject unknown parameters). - create_lvstore persists allow_device_colocation on raid_devices clusters. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/constants.py | 5 +++ simplyblock_core/rpc_client.py | 26 +++++++++-- .../services/migration_task_common.py | 27 ++++++++--- .../services/tasks_runner_failed_migration.py | 3 +- .../services/tasks_runner_migration.py | 1 + .../tasks_runner_new_dev_migration.py | 3 +- simplyblock_core/storage_node_ops.py | 6 +++ tests/unit/test_migration_relaxed_mode.py | 45 +++++++++++++++++-- 8 files changed, 101 insertions(+), 15 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 2f1a22ef9c..f102c3909e 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -1040,6 +1040,11 @@ def get_config_var(name, default=None): qos_class_replication_weight_percent = 10 MIG_PARALLEL_JOBS = 64 +# Send the placement anti-affinity floor to the data plane (affinity_floor on +# distr_migration_*_start, allow_device_colocation on bdev_distrib_create). +# Flip to True once the ultra feat/affinity-floor data plane is deployed; +# older data planes reject unknown RPC parameters. +DISTR_AFFINITY_FLOOR_RPC = False MIG_JOB_SIZE = 64 # Sync-replication resync (tasks_runner_sync_resync): the shared task driver diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index fa3bc542f3..b882af492f 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -343,6 +343,16 @@ def evict_cached_session(host, port) -> None: _session_pool.evict(host, port) + +def _add_affinity_floor(params: dict, affinity_floor) -> None: + """Add the placement anti-affinity floor ("device" or "relaxed", see + simplyblock_core.floors.migration_affinity_floor) to a migration start, + but only while constants.DISTR_AFFINITY_FLOOR_RPC is on: an older data + plane rejects an RPC with a parameter it does not know.""" + if affinity_floor and constants.DISTR_AFFINITY_FLOOR_RPC: + params["affinity_floor"] = affinity_floor + + class RPCClient: # ref: https://spdk.io/doc/jsonrpc.html @@ -1001,8 +1011,14 @@ def bdev_distrib_create(self, name, vuid, ndcs, npcs, num_blocks, block_size, jm distrib_cpu_mask="", ha_is_non_leader=True, jm_vuid=0, write_protection=False, full_page_unmap=True, shared_placement=False, write_protection_v2=False, page_copy=False, - synchronous_replication_mode=None, jm_n_local=None): + synchronous_replication_mode=None, jm_n_local=None, + allow_device_colocation: bool | None = None): """" + // Optional (cluster.raid_devices): several chunks of a stripe may share one + // device when no other device is writable. Sent only while + // constants.DISTR_AFFINITY_FLOOR_RPC is on (older data planes reject it). + "allow_device_colocation": true + // Optional, sync replication only: 0 disabled, 1 full. Not sent when None. "synchronous_replication_mode": 1 @@ -1064,6 +1080,8 @@ def bdev_distrib_create(self, name, vuid, ndcs, npcs, num_blocks, block_size, jm params["jm_n_local"] = jm_n_local if page_copy: params["page_copy"] = True + if allow_device_colocation is not None and constants.DISTR_AFFINITY_FLOOR_RPC: + params["allow_device_colocation"] = bool(allow_device_colocation) return self._request("bdev_distrib_create", **params) def distr_shared_placement(self, name=None, enable=True): @@ -1597,7 +1615,7 @@ def distr_migration_status(self, name): return self._request("distr_migration_status", name=name) def distr_migration_failure_start(self, name, storage_ID, qos_high_priority=False, job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS, - relaxed_mode=False): + relaxed_mode=False, affinity_floor: str | None = None): params = { "name": name, "storage_ID": storage_ID, @@ -1610,10 +1628,11 @@ def distr_migration_failure_start(self, name, storage_ID, qos_high_priority=Fals params["jobs"] = jobs if relaxed_mode: params["relaxed_mode"] = True + _add_affinity_floor(params, affinity_floor) return self._request("distr_migration_failure_start", **params) def distr_migration_expansion_start(self, name, qos_high_priority=False, job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS, - relaxed_mode=False, keep_placed=False): + relaxed_mode=False, keep_placed=False, affinity_floor: str | None = None): params = { "name": name, } @@ -1625,6 +1644,7 @@ def distr_migration_expansion_start(self, name, qos_high_priority=False, job_siz params["jobs"] = jobs if relaxed_mode: params["relaxed_mode"] = True + _add_affinity_floor(params, affinity_floor) # Page copy distribs only (older data planes reject the parameter): a balancing migration keeps # columns that page copies placed on another online device with anti-affinity kept. if keep_placed: diff --git a/simplyblock_core/services/migration_task_common.py b/simplyblock_core/services/migration_task_common.py index 46c8f2188b..0a90ef2543 100644 --- a/simplyblock_core/services/migration_task_common.py +++ b/simplyblock_core/services/migration_task_common.py @@ -9,7 +9,7 @@ import errno from datetime import UTC, datetime -from simplyblock_core import db_controller, storage_node_ops, utils +from simplyblock_core import db_controller, floors, storage_node_ops, utils from simplyblock_core.controllers import ( activation_gate, sync_replication_controller, @@ -491,13 +491,26 @@ def needs_relaxed_placement(member_nodes, ndcs, npcs): """Whether a migration on a cluster of ``member_nodes`` must run relaxed. Strict placement puts every chunk of a stripe on a different node. A - 1- or 2-node cluster, or one with fewer nodes than chunks per stripe, - has no such placement, so a strict migration has nowhere to move data - and never completes; relocation then needs a third node. ultra already - relaxes writes in that case on its own, but migrations only relax when - asked (``relaxed_mode``, default false). + 1- or 2-node cluster, or one with no more nodes than chunks per stripe, + has no such placement with a node to move to: with EXACTLY ndcs+npcs + member nodes every node already holds a chunk of every stripe, so a + strict migration away from one of them has nowhere to put the chunk + and never completes (a dead end once a node is down -- hence ``<=``, not + ``<``). ultra already relaxes writes in that case on its own, but + migrations only relax when asked (``relaxed_mode``, default false). """ - return member_nodes <= 2 or member_nodes < ndcs + npcs + return member_nodes <= 2 or member_nodes <= ndcs + npcs + + +def migration_affinity_floor(cluster): + """The data plane's placement anti-affinity floor for a migration on + ``cluster`` (a cluster record or its id): ``"relaxed"`` when its data + devices are protected below simplyblock (``raid_devices``), ``"device"`` + otherwise. Sent as ``affinity_floor`` beside ``relaxed_mode`` once + constants.DISTR_AFFINITY_FLOOR_RPC is on.""" + if isinstance(cluster, str): + cluster = db.get_cluster_by_id(cluster) + return floors.migration_affinity_floor(cluster) def migration_relaxed(cluster_id): diff --git a/simplyblock_core/services/tasks_runner_failed_migration.py b/simplyblock_core/services/tasks_runner_failed_migration.py index 6c850959c9..ea376bef85 100644 --- a/simplyblock_core/services/tasks_runner_failed_migration.py +++ b/simplyblock_core/services/tasks_runner_failed_migration.py @@ -45,7 +45,8 @@ def task_runner(task): task.function_params["distr_name"], device.cluster_device_order, mig.qos_high_priority(snode.cluster_id), job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS, - relaxed_mode=mig.migration_relaxed(snode.cluster_id))) + relaxed_mode=mig.migration_relaxed(snode.cluster_id), + affinity_floor=mig.migration_affinity_floor(snode.cluster_id))) if started is None: raise TaskAbort("canceled while starting migration") task = started diff --git a/simplyblock_core/services/tasks_runner_migration.py b/simplyblock_core/services/tasks_runner_migration.py index 6d2f9f7434..7885caaea4 100644 --- a/simplyblock_core/services/tasks_runner_migration.py +++ b/simplyblock_core/services/tasks_runner_migration.py @@ -108,6 +108,7 @@ def task_runner(task): task.function_params["distr_name"], mig.qos_high_priority(snode.cluster_id), job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS, relaxed_mode=mig.migration_relaxed(snode.cluster_id), + affinity_floor=mig.migration_affinity_floor(snode.cluster_id), keep_placed=bool(getattr(db.get_cluster_by_id(snode.cluster_id), "page_copy", False))), migration_devices=_online_device_count(task.cluster_id)) if started is None: diff --git a/simplyblock_core/services/tasks_runner_new_dev_migration.py b/simplyblock_core/services/tasks_runner_new_dev_migration.py index 8546965c5d..d8de61fb19 100644 --- a/simplyblock_core/services/tasks_runner_new_dev_migration.py +++ b/simplyblock_core/services/tasks_runner_new_dev_migration.py @@ -67,7 +67,8 @@ def task_runner(task): started = mig.start_migration(task, lambda: rpc_client.distr_migration_expansion_start( task.function_params["distr_name"], mig.qos_high_priority(snode.cluster_id), job_size=constants.MIG_JOB_SIZE, jobs=constants.MIG_PARALLEL_JOBS, - relaxed_mode=mig.migration_relaxed(snode.cluster_id))) + relaxed_mode=mig.migration_relaxed(snode.cluster_id), + affinity_floor=mig.migration_affinity_floor(snode.cluster_id))) if started is None: raise TaskAbort("canceled while starting migration") task = started diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 0876abcac8..0914135746 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -19237,6 +19237,12 @@ def create_lvstore(snode: StorageNode, ndcs, npcs, distr_bs, distr_chunk_bs, pag # creation, persisted per stack entry for the same reason. if cluster.page_copy: distrib_params["page_copy"] = True + # raid_devices: every data device is protected below simplyblock, so + # the distrib may put several chunks of a stripe on one device when no + # other device is writable. Persisted like the flags above; the RPC + # sends it only while constants.DISTR_AFFINITY_FLOOR_RPC is on. + if getattr(cluster, "raid_devices", False) is True: + distrib_params["allow_device_colocation"] = True # Failure-domain placement is activated implicitly via the per-node # failure_domain id (>= 0) in the distrib cluster map sent by # distr_send_cluster_map / distr_add_nodes (see get_distr_cluster_map); diff --git a/tests/unit/test_migration_relaxed_mode.py b/tests/unit/test_migration_relaxed_mode.py index 44a3e72b7d..3999a94861 100644 --- a/tests/unit/test_migration_relaxed_mode.py +++ b/tests/unit/test_migration_relaxed_mode.py @@ -28,10 +28,17 @@ def test_fewer_nodes_than_chunks_is_relaxed(self): self.assertTrue(mig.needs_relaxed_placement(5, 4, 2)) self.assertTrue(mig.needs_relaxed_placement(3, 2, 2)) + def test_exactly_as_many_nodes_as_chunks_is_relaxed(self): + """Every node holds a chunk of every stripe: a strict migration away + from one of them has nowhere to go (the migration dead end).""" + self.assertTrue(mig.needs_relaxed_placement(3, 2, 1)) + self.assertTrue(mig.needs_relaxed_placement(6, 4, 2)) + self.assertTrue(mig.needs_relaxed_placement(4, 2, 2)) + def test_enough_nodes_stay_strict(self): self.assertFalse(mig.needs_relaxed_placement(3, 1, 1)) - self.assertFalse(mig.needs_relaxed_placement(3, 2, 1)) - self.assertFalse(mig.needs_relaxed_placement(6, 4, 2)) + self.assertFalse(mig.needs_relaxed_placement(4, 2, 1)) + self.assertFalse(mig.needs_relaxed_placement(7, 4, 2)) class TestMigrationRelaxedLookup(unittest.TestCase): @@ -51,7 +58,7 @@ def test_removed_nodes_do_not_count(self): def test_offline_member_still_counts(self): cluster = mock.Mock(distr_ndcs=2, distr_npcs=1) nodes = self._nodes(StorageNode.STATUS_ONLINE, StorageNode.STATUS_ONLINE, - StorageNode.STATUS_OFFLINE) + StorageNode.STATUS_ONLINE, StorageNode.STATUS_OFFLINE) with mock.patch.object(mig, "db") as db: db.get_cluster_by_id.return_value = cluster db.get_storage_nodes_by_cluster_id.return_value = nodes @@ -79,6 +86,37 @@ def test_expansion_start_sends_flag_only_when_relaxed(self): c.distr_migration_expansion_start("d") self.assertNotIn("relaxed_mode", c._request.call_args.kwargs) + def test_affinity_floor_is_gated_by_the_constant(self): + """Older data planes reject unknown parameters: affinity_floor and + allow_device_colocation go out only once DISTR_AFFINITY_FLOOR_RPC is on.""" + c = self._client() + c.bdev_get = mock.Mock(return_value=None) + with mock.patch.object(rpc_mod.constants, "DISTR_AFFINITY_FLOOR_RPC", False): + c.distr_migration_failure_start("d", 3, affinity_floor="relaxed") + self.assertNotIn("affinity_floor", c._request.call_args.kwargs) + c.distr_migration_expansion_start("d", affinity_floor="device") + self.assertNotIn("affinity_floor", c._request.call_args.kwargs) + c.bdev_distrib_create("d", 1, 2, 1, 10, 4096, ["jm"], 4096, allow_device_colocation=True) + self.assertNotIn("allow_device_colocation", c._request.call_args.kwargs) + with mock.patch.object(rpc_mod.constants, "DISTR_AFFINITY_FLOOR_RPC", True): + c.distr_migration_failure_start("d", 3, affinity_floor="relaxed") + self.assertEqual(c._request.call_args.kwargs["affinity_floor"], "relaxed") + c.distr_migration_expansion_start("d", affinity_floor="device") + self.assertEqual(c._request.call_args.kwargs["affinity_floor"], "device") + c.distr_migration_expansion_start("d") + self.assertNotIn("affinity_floor", c._request.call_args.kwargs) + c.bdev_distrib_create("d", 1, 2, 1, 10, 4096, ["jm"], 4096, allow_device_colocation=True) + self.assertIs(c._request.call_args.kwargs["allow_device_colocation"], True) + c.bdev_distrib_create("d", 1, 2, 1, 10, 4096, ["jm"], 4096) + self.assertNotIn("allow_device_colocation", c._request.call_args.kwargs) + + def test_affinity_floor_follows_raid_devices(self): + self.assertEqual(mig.migration_affinity_floor(mock.Mock(raid_devices=True)), "relaxed") + self.assertEqual(mig.migration_affinity_floor(mock.Mock(raid_devices=False)), "device") + with mock.patch.object(mig, "db") as db: + db.get_cluster_by_id.return_value = mock.Mock(raid_devices=True) + self.assertEqual(mig.migration_affinity_floor("c1"), "relaxed") + class TestRunnersPassRelaxedMode(unittest.TestCase): """Every migration runner hands the cluster's relaxed decision to the RPC.""" @@ -94,6 +132,7 @@ def test_each_runner_passes_relaxed_mode(self): for mod in (tasks_runner_failed_migration, tasks_runner_migration, tasks_runner_new_dev_migration): src = inspect.getsource(mod) self.assertIn("relaxed_mode=mig.migration_relaxed(", src, mod.__name__) + self.assertIn("affinity_floor=mig.migration_affinity_floor(", src, mod.__name__) if __name__ == "__main__": From 6062e22a02e1d31948fe038e9d6f2feded43be17 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 12 Nov 2026 22:40:31 +0100 Subject: [PATCH 05/50] fix(lblk): default AIO bdev block size to 4096, keep recorded size on restart A new lblk device now gets a 4096-byte AIO base bdev; the device's logical block size is used only when 4096 is impossible (logical > 4096 or not a divisor, size not a 4K multiple, partition start not 4K aligned). 512e devices (EBS, most array LUNs) previously ended up with 512-byte bdevs. alceml's on-disk layout depends on the base bdev block size, so a device keeps the size it was formatted with: NVMeDevice.aio_block_size (0 = legacy, created without a block size, which keeps being re-created that way) is passed on every bdev_aio_create (addAioDevices at add-node and restart, restart_device, new_device_from_failed), and the bdev's block size is checked after creation. The size is also kept on the node's lblk_devices entry, which a control-plane service that predates the model field keeps on a read-modify-write; the restart path recovers it from there and the reconcile restores it on the device record. The node agent reports logical/physical block size and partition start in the block-device inventory. Without them (older agent) 4096 is chosen when the size allows it, and a new device whose 4096 create is refused is retried without a block size. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/constants.py | 8 + .../controllers/device_controller.py | 14 +- simplyblock_core/models/nvme_device.py | 9 + simplyblock_core/rpc_client.py | 4 +- simplyblock_core/storage_node_ops.py | 6 + simplyblock_core/utils/__init__.py | 156 ++++++- simplyblock_web/node_utils.py | 28 ++ tests/unit/test_lblk_aio_block_size.py | 382 ++++++++++++++++++ tests/unit/test_lblk_stable_paths.py | 2 +- 9 files changed, 600 insertions(+), 9 deletions(-) create mode 100644 tests/unit/test_lblk_aio_block_size.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 48a6c73b4a..3532ee4f52 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -108,6 +108,14 @@ def get_config_var(name, default=None): # Queue-depth sampling period enabled on every AIO base bdev so # bdev_get_iostat reports queue_depth (feeds the hung-IO watchdog). AIO_QD_SAMPLING_PERIOD_US = 100000 # 100 ms +# Block size of the AIO base bdev of a NEW lblk device. 512e devices (512 +# logical / 4096 physical, e.g. EBS and most array LUNs) would otherwise get +# 512-byte bdevs, which costs read-modify-write cycles in the device. The +# logical block size is used only when 4096 is not possible (see +# utils.choose_aio_block_size). An existing device always keeps the block +# size it was formatted with (NVMeDevice.aio_block_size): alceml's on-disk +# layout depends on it. +AIO_DEFAULT_BLOCK_SIZE = 4096 # Hung-IO watchdog: consecutive device_monitor polls (DEV_MONITOR_INTERVAL_SEC # apart) with queue_depth > 0 and zero completion progress before the device # is declared stalled (3 x 10s = 30s — deliberately above kernel SCSI/NVMe diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index d8317fd5f6..d1f6e35d23 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -817,7 +817,12 @@ def restart_device(device_id, force=False): if not filename: logger.error(f"No block device path known for {device_id}") return False - snode.rpc_client().bdev_aio_create(device_obj.nvme_bdev, filename) + # The block size the device was formatted with (0: legacy). + block_size = (device_obj.aio_block_size + or utils.recorded_aio_block_size(snode, device_obj.serial_number) + or 0) + utils.create_aio_bdev(snode.rpc_client(), device_obj.nvme_bdev, + filename, block_size) snode.rpc_client().bdev_examine(device_obj.nvme_bdev) snode.rpc_client().bdev_wait_for_examine() snode.rpc_client().bdev_set_qd_sampling_period( @@ -1655,7 +1660,12 @@ def new_device_from_failed(device_id): if not filename: logger.error(f"No block device path known for {device_id}") return False - device_node.rpc_client().bdev_aio_create(device.nvme_bdev, filename) + # The block size the device was formatted with (0: legacy). + block_size = (device.aio_block_size + or utils.recorded_aio_block_size(device_node, device.serial_number) + or 0) + utils.create_aio_bdev(device_node.rpc_client(), device.nvme_bdev, + filename, block_size) device_node.rpc_client().bdev_examine(device.nvme_bdev) device_node.rpc_client().bdev_wait_for_examine() device_node.rpc_client().bdev_set_qd_sampling_period( diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index 54e815cb23..2dfd5310b5 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -113,6 +113,15 @@ class NVMeDevice(BaseModel): # Stable /dev/disk/by-id/... symlink when the device has one; preferred # as the AIO bdev filename so udev renames cannot bite mid-flight. by_id_path: str = "" + # Block size in bytes the AIO base bdev of an lblk device was created + # with, and must be re-created with on every restart: alceml's on-disk + # layout depends on the base bdev block size, so data written at one + # size cannot be read at another. 0 means the bdev was created without a + # block size (SPDK used the device's logical block size): every device + # created before this field existed, which keeps being created that way. + # Also kept in the node's ``lblk_devices`` entries (see + # utils.recorded_aio_block_size). + aio_block_size: int = 0 # The latest disk-health assessment an external assessor (the Kubernetes # operator) recorded through PUT .../devices/{id}/health-assessment, # stored verbatim: {"reporter", "reported_at", "valid_seconds", diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 71c46ba368..8648a0b631 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1460,7 +1460,9 @@ def bdev_aio_create(self, name, filename, block_size=0): """Create an SPDK AIO bdev over a Linux block device (lblk cluster mode). ``filename`` is the device path — prefer the stable /dev/disk/by-id symlink. ``block_size`` 0 lets SPDK use the device's - logical block size.""" + logical block size. Callers go through utils.create_aio_bdev, which + passes the block size a device was formatted with + (NVMeDevice.aio_block_size).""" kwargs = {"name": name, "filename": filename} if block_size: kwargs["block_size"] = block_size diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index c276cbd72a..ea35ecedd2 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -1757,6 +1757,7 @@ def _create_jm_stack_on_device(rpc_client, nvme, snode: StorageNode, after_resta 'alceml_name': alceml_name, 'nvme_bdev': nvme.nvme_bdev, "serial_number": nvme.serial_number, + 'aio_block_size': nvme.aio_block_size, "device_data_dict": nvme.to_dict(), 'jm_bdev': jm_bdev, 'testing_bdev': test_name, @@ -8505,6 +8506,11 @@ def _reconcile_discovered_devices(snode, nvme_devs): db_dev.bdev_type = "aio" db_dev.device_path = found_dev.device_path db_dev.by_id_path = found_dev.by_id_path + # Restore a block size a control-plane service that + # predates the field dropped from the record (found_dev + # got it from the node's lblk_devices entry). + if found_dev.aio_block_size and not db_dev.aio_block_size: + db_dev.aio_block_size = found_dev.aio_block_size else: db_dev.nvme_controller = found_dev.nvme_controller db_dev.pcie_address = found_dev.pcie_address diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 7f6ea8970f..492e08001a 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1607,10 +1607,142 @@ def resolve_lblk_entries(configured_entries, host_devices): resolved_entry["partuuid"] = live.get("partuuid") or entry.get("partuuid", "") if entry.get("journal"): resolved_entry["journal"] = True + # Block-size inputs for a new device's AIO bdev (choose_aio_block_size). + # An agent that predates them sends none; the chooser copes. + for key in ("logical_block_size", "physical_block_size", "start_bytes"): + if live.get(key) is not None: + resolved_entry[key] = live[key] + # The block size this device was formatted with, kept on the config + # entry as a second copy of NVMeDevice.aio_block_size. Only for the + # very same device: a different device in the same slot is new. + if entry.get("aio_block_size") and resolved_entry["serial"] == entry.get("serial"): + resolved_entry["aio_block_size"] = int(entry["aio_block_size"]) resolved.append(resolved_entry) return resolved, missing +def choose_aio_block_size(entry) -> int: + """Block size for the AIO bdev of a NEW lblk device (inventory or + resolved entry): AIO_DEFAULT_BLOCK_SIZE (4096) whenever the device allows + it, the device's logical block size otherwise. 4096 is possible when it is + a multiple of the logical block size, the device size is a multiple of it, + and, for a partition whose start is known, the start is 4096-aligned. + + An agent that predates ``logical_block_size`` does not send it. 4096 is + still chosen then when the size allows it: every logical block size up to + 4096 divides it, so SPDK accepts it, and addAioDevices retries a new + device without a block size if SPDK refuses (logical block size above + 4096). When 4096 is not possible, 0 (SPDK picks the logical block size) is + returned instead of a guessed 512, which SPDK would refuse on a 4Kn + device. Never used for a device that already has a record: that one keeps + the block size it was formatted with (recorded_aio_block_size).""" + target = constants.AIO_DEFAULT_BLOCK_SIZE + try: + logical = int(entry.get("logical_block_size") or 0) + except (TypeError, ValueError): + logical = 0 + fallback = max(0, logical) + if logical > target or (logical > 0 and target % logical): + return fallback + size = int(entry.get("size") or 0) + if size <= 0 or size % target: + return fallback + start = entry.get("start_bytes") + if start is not None and int(start) % target: + return fallback + return target + + +def recorded_aio_block_size(snode, serial): + """The block size the AIO bdev of the device with this serial was created + with, from what the control plane recorded about it: its NVMeDevice + record, the node's journal device record, and the node's ``lblk_devices`` + config entry (a plain dict, which a control-plane service that predates + NVMeDevice.aio_block_size keeps intact on a read-modify-write of the + node, while it drops the unknown model field). + + Returns the recorded size, 0 for a device recorded before block sizes were + (legacy: create without a block size, as it was formatted), or None when + nothing is recorded: a new device, which gets choose_aio_block_size.""" + known = False + sizes = [] + devices = getattr(snode, "nvme_devices", None) + for dev in devices if isinstance(devices, list) else []: + if getattr(dev, "serial_number", None) == serial: + known = True + sizes.append(int(getattr(dev, "aio_block_size", 0) or 0)) + jm = getattr(snode, "jm_device", None) + if jm is not None: + data = getattr(jm, "device_data_dict", None) + data = data if isinstance(data, dict) else {} + if data.get("serial_number") == serial or getattr(jm, "serial_number", None) == serial: + known = True + sizes.append(int(data.get("aio_block_size") or 0)) + sizes.append(int(getattr(jm, "aio_block_size", 0) or 0)) + entries = getattr(snode, "lblk_devices", None) + for entry in entries if isinstance(entries, list) else []: + if isinstance(entry, dict) and entry.get("serial") == serial: + sizes.append(int(entry.get("aio_block_size") or 0)) + recorded = next((s for s in sizes if s > 0), 0) + if recorded: + conflicting = sorted({s for s in sizes if s > 0 and s != recorded}) + if conflicting: + logger.warning(f"Device {serial}: conflicting recorded AIO block sizes " + f"{[recorded] + conflicting}; using {recorded}") + return recorded + return 0 if known else None + + +def remember_aio_block_size(snode, serial, block_size): + """Store the block size of a device's AIO bdev on the node's + ``lblk_devices`` entry, next to NVMeDevice.aio_block_size (see + recorded_aio_block_size). 0 (legacy) is never written.""" + entries = getattr(snode, "lblk_devices", None) + if not block_size or not isinstance(entries, list): + return + for entry in entries: + if isinstance(entry, dict) and entry.get("serial") == serial: + entry["aio_block_size"] = int(block_size) + + +def check_aio_block_size(name, bdev, block_size): + """Raise when the AIO bdev does not have the expected (non-zero) block + size. A bdev that reports no block size is not checked.""" + if not block_size or not isinstance(bdev, dict): + return + actual = bdev.get("block_size") + if isinstance(actual, int) and actual != block_size: + raise Exception(f"AIO bdev {name} has block size {actual}, expected " + f"{block_size}; refusing to use it as storage") + + +def create_aio_bdev(rpc_client, name, filename, block_size=0, new_device=False, + verify=True): + """bdev_aio_create with ``block_size`` (0: none, SPDK uses the logical + block size), then (``verify``) check that the bdev has that block size; + a caller that reads the bdev anyway passes verify=False and runs + check_aio_block_size itself. Returns the block size requested. Raises + when the bdev cannot be created or its block size differs from the one + requested: building alceml over a bdev of another block size would make + the device's data unreadable. Only for a ``new_device`` (nothing + formatted on it yet) a refused block size is retried without one.""" + if block_size: + ret = rpc_client.bdev_aio_create(name, filename, block_size=block_size) + if not ret and new_device: + logger.warning(f"bdev_aio_create {name} ({filename}) with block size " + f"{block_size} failed on {rpc_client.host}; retrying with " + f"the device's logical block size") + block_size = 0 + ret = rpc_client.bdev_aio_create(name, filename) + else: + ret = rpc_client.bdev_aio_create(name, filename) + if not ret: + raise Exception(f"bdev_aio_create failed for {name} ({filename}) on {rpc_client.host}") + if verify: + check_aio_block_size(name, rpc_client.bdev_get(name), block_size) + return block_size + + def addAioDevices(rpc_client, snode, blk_entries): """lblk-mode sibling of addNvmeDevices: create one SPDK AIO bdev per resolved block device and model it as an NVMeDevice with @@ -1621,27 +1753,40 @@ def addAioDevices(rpc_client, snode, blk_entries): next_physical_label = snode.physical_label for entry in blk_entries: bdev_name = aio_bdev_name_for_serial(entry["serial"]) + # A device the cluster knows is re-created with the block size it was + # formatted with (0: legacy, none); only a new one gets the default. + recorded = recorded_aio_block_size(snode, entry["serial"]) + if recorded is None and entry.get("aio_block_size"): + recorded = int(entry["aio_block_size"]) ret = rpc_client.bdev_get(bdev_name) if not ret: # A path the caller selected the device by is opened as given. # Otherwise prefer the by-id path as the filename so a udev rename # between resolution and create cannot swap devices under us. filename = entry.get("path") or entry.get("by_id") or entry["current_path"] - ret = rpc_client.bdev_aio_create(bdev_name, filename) - if not ret: - raise Exception( - f"bdev_aio_create failed for {bdev_name} ({filename}) " - f"on {rpc_client.host}") + if recorded is None: + expected = create_aio_bdev(rpc_client, bdev_name, filename, + choose_aio_block_size(entry), + new_device=True, verify=False) + else: + expected = create_aio_bdev(rpc_client, bdev_name, filename, + recorded, verify=False) + else: + expected = recorded rpc_client.bdev_examine(bdev_name) rpc_client.bdev_wait_for_examine() bdev = rpc_client.bdev_get(bdev_name) if not bdev: raise Exception(f"AIO bdev {bdev_name} not found after create on {rpc_client.host}") + check_aio_block_size(bdev_name, bdev, expected) total_size = bdev['block_size'] * bdev['num_blocks'] if total_size == 0: logger.warning(f"Skipping zero-size block device {entry['name']} ({bdev_name})") continue + # A new device is formatted at the block size its bdev has now. + aio_block_size = int(bdev['block_size']) if recorded is None else recorded + remember_aio_block_size(snode, entry["serial"], aio_block_size) # Queue-depth sampling feeds the control-plane hung-IO watchdog # (AIO has no bdev_nvme-style timeout/action_on_timeout). @@ -1663,6 +1808,7 @@ def addAioDevices(rpc_client, snode, blk_entries): 'nvme_bdev': bdev_name, 'nvme_controller': "", 'bdev_type': "aio", + 'aio_block_size': aio_block_size, 'device_path': entry["current_path"], 'by_id_path': entry.get("path") or entry.get("by_id", ""), 'node_id': snode.get_id(), diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index 1abefb2559..07dbc7bccc 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -350,6 +350,10 @@ def get_block_devices_info() -> list[dict]: serial = "SYN-" + hashlib.sha1(seed.encode()).hexdigest()[:16] synthetic = True numa_node = int(_read_sysfs(f"/sys/block/{name}/device/numa_node") or -1) + # The control plane picks the AIO bdev block size of a new device + # from these (utils.choose_aio_block_size): 4096 unless impossible. + logical_bs = _logical_block_size(name) + physical_bs = _physical_block_size(name) devices.append({ "name": name, "device_path": f"/dev/{name}", @@ -369,6 +373,8 @@ def get_block_devices_info() -> list[dict]: "by_id_path": by_id_path, "by_id_paths": by_id_paths, "numa_node": numa_node, + "logical_block_size": logical_bs, + "physical_block_size": physical_bs, }) for child in children: if child.get("type") != "part": @@ -405,6 +411,9 @@ def get_block_devices_info() -> list[dict]: "by_id_path": part_paths[0] if part_paths else "", "by_id_paths": part_paths, "numa_node": numa_node, + "logical_block_size": logical_bs, + "physical_block_size": physical_bs, + "start_bytes": _partition_start_bytes(name, part_name), }) logger.debug("function:get_block_devices_info end") return devices @@ -448,6 +457,25 @@ def _logical_block_size(disk_name: str) -> int: return size if size > 0 else 512 +def _physical_block_size(disk_name: str) -> int: + """Physical sector size of a disk, in bytes (informational: a 512e + device reports 512 logical / 4096 physical).""" + try: + size = int(_read_sysfs(f"/sys/block/{disk_name}/queue/physical_block_size") or 0) + except ValueError: + size = 0 + return size if size > 0 else _logical_block_size(disk_name) + + +def _partition_start_bytes(disk_name: str, part_name: str) -> int | None: + """Byte offset of a partition on its disk; sysfs counts the start in + 512-byte units whatever the logical block size. None when unreadable.""" + try: + return int(_read_sysfs(f"/sys/block/{disk_name}/{part_name}/start")) * 512 + except ValueError: + return None + + def split_partition_partner(part: dict, devices: list[dict]) -> tuple[dict, dict] | None: """The ``(journal, data)`` pair a previous split_partition_for_journal made of a partition, when ``part`` is one of the two; otherwise None. diff --git a/tests/unit/test_lblk_aio_block_size.py b/tests/unit/test_lblk_aio_block_size.py new file mode 100644 index 0000000000..0c52c49f52 --- /dev/null +++ b/tests/unit/test_lblk_aio_block_size.py @@ -0,0 +1,382 @@ +"""Unit tests for the block size of lblk AIO base bdevs. + +A new device gets a 4096-byte AIO bdev unless 4096 is impossible, then the +device's logical block size. A device that already has a record is always +re-created with the block size it was formatted with, because alceml's +on-disk layout depends on it: 0 (legacy, created before the block size was +recorded) means "no block size", exactly as it was created. + +Covered: + - utils.choose_aio_block_size: 512e, 512n with a non-4K size, 4Kn, logical + above 4096, an agent that does not send logical_block_size, partition + start alignment. + - utils.addAioDevices: new device -> chosen size recorded on the device and + on the node's lblk_devices entry; legacy record -> no block size; 4096 + record -> 4096 whatever the inventory says; record dropped by an older + control-plane service -> recovered from the lblk_devices entry; journal + device record; SPDK refusing 4096 for a new device -> retried without; + block-size mismatch -> refused. + - utils.resolve_lblk_entries carries the block-size fields. + - storage_node_ops._reconcile_discovered_devices restores a dropped size. + - device_controller.restart_device / new_device_from_failed pass the + recorded size. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import utils +from simplyblock_core.controllers import device_controller +from simplyblock_core.models.nvme_device import JMDevice, NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.storage_node_ops import _reconcile_discovered_devices + +GIB = 1 << 30 + + +def _entry(serial="S1", size=100 * GIB, logical=512, physical=4096, **extra): + e = {"name": "sdb", "serial": serial, "by_id": f"/dev/disk/by-id/wwn-{serial}", + "size": size, "numa": 0, "model": "M", "current_path": "/dev/sdb", + "has_partitions": False} + if logical is not None: + e["logical_block_size"] = logical + if physical is not None: + e["physical_block_size"] = physical + e.update(extra) + return e + + +def _snode(devices=(), lblk_devices=None, jm_device=None): + n = StorageNode() + n.uuid = "node-1" + n.cluster_id = "cluster-1" + n.nvme_devices = list(devices) + n.lblk_devices = lblk_devices if lblk_devices is not None else [ + {"name": "sdb", "serial": "S1", "size": 100 * GIB}] + if jm_device is not None: + n.jm_device = jm_device + return n + + +def _record(serial="S1", aio_block_size=0, status=NVMeDevice.STATUS_ONLINE): + d = NVMeDevice() + d.uuid = f"dev-{serial}" + d.serial_number = serial + d.bdev_type = "aio" + d.nvme_bdev = utils.aio_bdev_name_for_serial(serial) + d.aio_block_size = aio_block_size + d.status = status + return d + + +class _FakeRpc: + """SPDK fake modelling bdev_aio_create's block-size rules: no block size + -> the logical block size; one below the logical block size -> refused.""" + + def __init__(self, logical=512, num_bytes=100 * GIB, existing=None, + ignore_block_size=False): + self.host = "test-host" + self.logical = logical + self.num_bytes = num_bytes + self.bdevs = dict(existing or {}) + self.ignore_block_size = ignore_block_size + self.creates = [] + + def bdev_get(self, name): + return self.bdevs.get(name) + + def bdev_aio_create(self, name, filename, block_size=0): + self.creates.append(block_size) + if block_size and block_size < self.logical: + return None + bs = self.logical if (not block_size or self.ignore_block_size) else block_size + self.bdevs[name] = {"name": name, "block_size": bs, + "num_blocks": self.num_bytes // bs} + return name + + def bdev_examine(self, name): + return True + + def bdev_wait_for_examine(self): + return True + + def bdev_set_qd_sampling_period(self, name, period): + return True + + +class TestChooseAioBlockSize(unittest.TestCase): + + def test_512e_gets_4096(self): + self.assertEqual(utils.choose_aio_block_size(_entry(logical=512, physical=4096)), 4096) + + def test_512n_with_non_4k_size_gets_512(self): + self.assertEqual(utils.choose_aio_block_size( + _entry(logical=512, physical=512, size=100 * GIB + 512)), 512) + + def test_512n_with_4k_size_gets_4096(self): + self.assertEqual(utils.choose_aio_block_size(_entry(logical=512, physical=512)), 4096) + + def test_4kn_gets_4096(self): + self.assertEqual(utils.choose_aio_block_size(_entry(logical=4096, physical=4096)), 4096) + + def test_logical_above_4096_kept(self): + self.assertEqual(utils.choose_aio_block_size(_entry(logical=8192)), 8192) + + def test_agent_without_field_4k_size_gets_4096(self): + self.assertEqual(utils.choose_aio_block_size(_entry(logical=None, physical=None)), 4096) + + def test_agent_without_field_odd_size_leaves_it_to_spdk(self): + self.assertEqual(utils.choose_aio_block_size( + _entry(logical=None, physical=None, size=100 * GIB + 512)), 0) + + def test_unaligned_partition_start_gets_logical(self): + self.assertEqual(utils.choose_aio_block_size( + _entry(type="part", start_bytes=63 * 512)), 512) + + def test_aligned_partition_start_gets_4096(self): + self.assertEqual(utils.choose_aio_block_size( + _entry(type="part", start_bytes=2048 * 512)), 4096) + + +class TestAddAioDevicesBlockSize(unittest.TestCase): + + def test_new_512e_device_created_at_4096_and_recorded(self): + rpc = _FakeRpc(logical=512) + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, [4096]) + self.assertEqual(devs[0].aio_block_size, 4096) + self.assertEqual(snode.lblk_devices[0]["aio_block_size"], 4096) + + def test_new_512n_odd_size_created_at_512(self): + rpc = _FakeRpc(logical=512, num_bytes=100 * GIB + 512) + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_entry(physical=512, size=100 * GIB + 512)]) + self.assertEqual(rpc.creates, [512]) + self.assertEqual(devs[0].aio_block_size, 512) + self.assertEqual(snode.lblk_devices[0]["aio_block_size"], 512) + + def test_new_4kn_device_created_at_4096(self): + rpc = _FakeRpc(logical=4096) + devs = utils.addAioDevices(rpc, _snode(), [_entry(logical=4096)]) + self.assertEqual(rpc.creates, [4096]) + self.assertEqual(devs[0].aio_block_size, 4096) + + def test_legacy_record_restarted_without_block_size(self): + rpc = _FakeRpc(logical=512) + snode = _snode(devices=[_record(aio_block_size=0)]) + devs = utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, [0]) + self.assertEqual(devs[0].aio_block_size, 0) + self.assertNotIn("aio_block_size", snode.lblk_devices[0]) + + def test_4096_record_restarted_at_4096(self): + rpc = _FakeRpc(logical=512) + snode = _snode(devices=[_record(aio_block_size=4096)]) + # Whatever the inventory says now, the recorded size wins. + utils.addAioDevices(rpc, snode, [_entry(size=100 * GIB + 512)]) + self.assertEqual(rpc.creates, [4096]) + + def test_512_record_restarted_at_512(self): + rpc = _FakeRpc(logical=512) + snode = _snode(devices=[_record(aio_block_size=512)]) + utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, [512]) + + def test_size_dropped_from_record_recovered_from_lblk_entry(self): + # An older control-plane service rewrote the node and dropped + # NVMeDevice.aio_block_size; the lblk_devices dict kept it. + rpc = _FakeRpc(logical=512) + snode = _snode(devices=[_record(aio_block_size=0)], + lblk_devices=[{"name": "sdb", "serial": "S1", "aio_block_size": 4096}]) + devs = utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, [4096]) + self.assertEqual(devs[0].aio_block_size, 4096) + + def test_journal_device_record_used(self): + jm = JMDevice() + jm.serial_number = "S1" + jm.device_data_dict = {"serial_number": "S1", "aio_block_size": 4096} + rpc = _FakeRpc(logical=512) + utils.addAioDevices(rpc, _snode(jm_device=jm), [_entry(size=100 * GIB + 512)]) + self.assertEqual(rpc.creates, [4096]) + + def test_legacy_journal_device_record_without_block_size(self): + jm = JMDevice() + jm.serial_number = "S1" + jm.device_data_dict = {"serial_number": "S1"} + rpc = _FakeRpc(logical=512) + utils.addAioDevices(rpc, _snode(jm_device=jm), [_entry()]) + self.assertEqual(rpc.creates, [0]) + + def test_agent_without_field_new_device(self): + rpc = _FakeRpc(logical=512) + devs = utils.addAioDevices(rpc, _snode(), [_entry(logical=None, physical=None)]) + self.assertEqual(rpc.creates, [4096]) + self.assertEqual(devs[0].aio_block_size, 4096) + + def test_new_device_refused_4096_retried_without(self): + rpc = _FakeRpc(logical=8192) + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_entry(logical=None, physical=None)]) + self.assertEqual(rpc.creates, [4096, 0]) + self.assertEqual(devs[0].aio_block_size, 8192) + self.assertEqual(snode.lblk_devices[0]["aio_block_size"], 8192) + + def test_recorded_device_refused_is_not_retried(self): + rpc = _FakeRpc(logical=8192) + snode = _snode(devices=[_record(aio_block_size=4096)]) + with self.assertRaisesRegex(Exception, "bdev_aio_create failed"): + utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, [4096]) + + def test_spdk_ignoring_block_size_refused(self): + rpc = _FakeRpc(logical=512, ignore_block_size=True) + snode = _snode(devices=[_record(aio_block_size=4096)]) + with self.assertRaisesRegex(Exception, "expected 4096"): + utils.addAioDevices(rpc, snode, [_entry()]) + + def test_existing_bdev_of_other_size_refused(self): + name = utils.aio_bdev_name_for_serial("S1") + rpc = _FakeRpc(existing={name: {"name": name, "block_size": 512, "num_blocks": 10}}) + snode = _snode(devices=[_record(aio_block_size=4096)]) + with self.assertRaisesRegex(Exception, "expected 4096"): + utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates, []) + + def test_existing_bdev_of_new_device_recorded_at_its_size(self): + name = utils.aio_bdev_name_for_serial("S1") + rpc = _FakeRpc(existing={name: {"name": name, "block_size": 512, "num_blocks": 10}}) + devs = utils.addAioDevices(rpc, _snode(), [_entry()]) + self.assertEqual(rpc.creates, []) + self.assertEqual(devs[0].aio_block_size, 512) + + +class TestResolveCarriesBlockSize(unittest.TestCase): + + def _live(self, serial): + return {"name": "sdx", "device_path": "/dev/sdx", "serial": serial, + "by_id_path": "", "size": 100 * GIB, "numa_node": 0, + "logical_block_size": 512, "physical_block_size": 4096, + "type": "disk"} + + def test_fields_carried_for_same_device(self): + configured = [{"name": "sdb", "serial": "S1", "size": 1, "aio_block_size": 4096}] + resolved, _ = utils.resolve_lblk_entries(configured, [self._live("S1")]) + self.assertEqual(resolved[0]["logical_block_size"], 512) + self.assertEqual(resolved[0]["physical_block_size"], 4096) + self.assertEqual(resolved[0]["aio_block_size"], 4096) + + def test_recorded_size_not_carried_to_a_different_device(self): + configured = [{"name": "sdx", "serial": "S1", "size": 1, "aio_block_size": 512}] + resolved, _ = utils.resolve_lblk_entries(configured, [self._live("OTHER")]) + self.assertNotIn("aio_block_size", resolved[0]) + + +class TestReconcileRestoresBlockSize(unittest.TestCase): + + def test_dropped_size_restored(self): + db_dev = _record(aio_block_size=0) + found = _record(aio_block_size=4096) + snode = _snode(devices=[db_dev]) + _reconcile_discovered_devices(snode, [found]) + self.assertEqual(db_dev.aio_block_size, 4096) + + def test_recorded_size_never_overwritten(self): + db_dev = _record(aio_block_size=512) + found = _record(aio_block_size=4096) + _reconcile_discovered_devices(_snode(devices=[db_dev]), [found]) + self.assertEqual(db_dev.aio_block_size, 512) + + +class TestDeviceControllerBlockSize(unittest.TestCase): + + def _restart(self, aio_block_size, lblk_devices=None): + device = _record(aio_block_size=aio_block_size, status=NVMeDevice.STATUS_REMOVED) + device.node_id = "node-1" + device.cluster_id = "cluster-1" + device.by_id_path = "/dev/disk/by-id/wwn-1" + device.nvmf_nqn = "" + device.alceml_bdev = "" + snode = MagicMock() + snode.cluster_id = "cluster-1" + snode.nvme_devices = [device] + snode.jm_device = None + snode.lblk_devices = lblk_devices or [] + + rpc = MagicMock() + state = {"present": False} + rpc.bdev_get.side_effect = ( + lambda name: {"name": name, "block_size": state.get("bs", 512)} + if state["present"] else None) + + def _aio_create(name, filename, block_size=0): + state["present"] = True + state["bs"] = block_size or 512 + return name + + rpc.bdev_aio_create.side_effect = _aio_create + snode.rpc_client.return_value = rpc + snode.client.return_value.get_blockdevices.return_value = ([], None) + + db = MagicMock() + db.get_storage_device_by_id.return_value = device + db.get_storage_node_by_id.return_value = snode + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(device_controller.tasks_controller, + "get_active_dev_restart_task", return_value=None), \ + patch.object(device_controller, "device_set_retries_exhausted"), \ + patch.object(device_controller, "device_set_unavailable"), \ + patch.object(device_controller, "_def_create_device_stack", return_value=True), \ + patch.object(device_controller, "device_set_io_error"), \ + patch.object(device_controller, "device_set_online"), \ + patch.object(device_controller, "device_events"): + result = device_controller.restart_device(device.uuid) + return result, rpc + + def test_restart_legacy_device_without_block_size(self): + result, rpc = self._restart(0) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with("aio_S1", "/dev/disk/by-id/wwn-1") + + def test_restart_4096_device_at_4096(self): + result, rpc = self._restart(4096) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + + def test_restart_uses_lblk_entry_when_record_dropped_it(self): + result, rpc = self._restart(0, lblk_devices=[{"serial": "S1", "aio_block_size": 4096}]) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + + def test_new_device_from_failed_keeps_block_size(self): + device = _record(aio_block_size=4096, status=NVMeDevice.STATUS_FAILED_AND_MIGRATED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + node = MagicMock() + node.get_id.return_value = "node-1" + node.nvme_devices = [device] + node.lblk_devices = [] + rpc = MagicMock() + state = {"present": False} + rpc.bdev_get.side_effect = ( + lambda name: {"name": name, "block_size": 4096} if state["present"] else None) + + def _aio_create(name, filename, block_size=0): + state["present"] = True + return name + + rpc.bdev_aio_create.side_effect = _aio_create + node.rpc_client.return_value = rpc + node.client.return_value.get_blockdevices.return_value = ([], None) + db = MagicMock() + db.get_storage_node_by_device_id.return_value = node + with patch.object(device_controller, "DBController", return_value=db): + self.assertTrue(device_controller.new_device_from_failed(device.uuid)) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_lblk_stable_paths.py b/tests/unit/test_lblk_stable_paths.py index d44ce3604d..0be129dc5c 100644 --- a/tests/unit/test_lblk_stable_paths.py +++ b/tests/unit/test_lblk_stable_paths.py @@ -184,7 +184,7 @@ def test_the_path_is_the_filename_and_the_recorded_by_id_path(self): devices = utils.addAioDevices(rpc, self._snode(), resolved) rpc.bdev_aio_create.assert_called_once_with( - utils.aio_bdev_name_for_serial("S-DATA"), DATA_BY_ID_ALT) + utils.aio_bdev_name_for_serial("S-DATA"), DATA_BY_ID_ALT, block_size=4096) self.assertEqual(devices[0].by_id_path, DATA_BY_ID_ALT) From 9783742cae50046160482a20254371e827f41929 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 10:16:12 +0100 Subject: [PATCH 06/50] feat(lblk): serve device-mapper nodes through bdev_uring SPDK's bdev_aio submits every I/O on a block device with RWF_NOWAIT so io_submit never blocks the reactor. device-mapper honours RWF_NOWAIT only when every target does; dm-multipath does not, so every AIO submit to a multipath map fails with EOPNOTSUPP and alceml cannot even format it. Dropping RWF_NOWAIT is not an option (a blocking submit during a path failover would freeze the reactor), so device-mapper nodes are served by SPDK's io_uring bdev instead. Each lblk device now has an I/O engine, NVMeDevice.lblk_io_engine ("aio" default, or "uring"), also kept on the node's lblk_devices entry next to aio_block_size. Rule (utils.lblk_io_engine_for): a kernel name starting with dm- (including a kpartx partition of a multipath map) or an lsblk TYPE in mpath/lvm/crypt/dm gets uring; whole disks, sd/nvme partitions and md arrays stay on aio. For an agent that predates the fields, a /dev/dm-, /dev/mapper/ or by-id dm-name-/dm-uuid- path still selects uring. The recorded engine is kept, except that the rule's uring wins over a recorded aio (aio cannot serve the device; logged). The engine does not change the on-disk format. The node agent reports KNAME in the block-device inventory; resolve_lblk_entries carries kname, type, the derived engine and the recorded engine. rpc_client gains bdev_uring_create/delete/rescan, and every create (addAioDevices, restart_device, new_device_from_failed) and the growth rescan go through engine-aware helpers (create_aio_bdev with io_engine, rescan_lblk_bdev, delete_lblk_bdev). The bdev name (aio_) and bdev_type "aio" are unchanged, and the block-size rules apply to both engines. An existing bdev's engine is taken from its driver_specific info. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/constants.py | 15 + .../controllers/device_controller.py | 25 +- simplyblock_core/models/nvme_device.py | 7 + simplyblock_core/rpc_client.py | 17 + simplyblock_core/storage_node_ops.py | 4 + simplyblock_core/utils/__init__.py | 216 +++++++- simplyblock_web/node_utils.py | 7 +- tests/unit/test_lblk_io_engine.py | 470 ++++++++++++++++++ 8 files changed, 739 insertions(+), 22 deletions(-) create mode 100644 tests/unit/test_lblk_io_engine.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 3532ee4f52..d74cab1588 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -116,6 +116,21 @@ def get_config_var(name, default=None): # size it was formatted with (NVMeDevice.aio_block_size): alceml's on-disk # layout depends on it. AIO_DEFAULT_BLOCK_SIZE = 4096 +# I/O engine of an lblk device's base bdev (NVMeDevice.lblk_io_engine). +# SPDK's bdev_aio submits every I/O on a block device with RWF_NOWAIT so +# io_submit never blocks the reactor. device-mapper honours RWF_NOWAIT only +# when every target in the table does; dm-multipath does not, so every AIO +# submit to a multipath map fails with EOPNOTSUPP. Device-mapper devices are +# therefore served by SPDK's io_uring bdev (bdev_uring_create), everything +# else (whole disks, sd/nvme partitions, md arrays) by bdev_aio. The bdev +# NAME (aio_) and NVMeDevice.bdev_type "aio" are the same for both. +LBLK_IO_ENGINE_AIO = "aio" +LBLK_IO_ENGINE_URING = "uring" +LBLK_IO_ENGINES = (LBLK_IO_ENGINE_AIO, LBLK_IO_ENGINE_URING) +# lsblk TYPE values of device-mapper nodes (multipath map, LVM volume, +# dm-crypt, other dm). A partition mapped from a multipath map (kpartx) has +# TYPE "part" but a dm-* kernel name. +LBLK_DM_TYPES = ("mpath", "lvm", "crypt", "dm") # Hung-IO watchdog: consecutive device_monitor polls (DEV_MONITOR_INTERVAL_SEC # apart) with queue_depth > 0 and zero completion progress before the device # is declared stalled (3 x 10s = 30s — deliberately above kernel SCSI/NVMe diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index d1f6e35d23..6d517ff9f9 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -804,10 +804,12 @@ def restart_device(device_id, force=False): if not snode.rpc_client().bdev_get(device_obj.nvme_bdev): try: filename = device_obj.by_id_path or device_obj.device_path + live = None try: inventory, _ = snode.client(timeout=30, retry=1).get_blockdevices() for blk in inventory or []: if blk.get("serial") == device_obj.serial_number: + live = blk filename = blk.get("by_id_path") or blk.get("device_path") device_obj.device_path = blk.get("device_path", device_obj.device_path) device_obj.by_id_path = blk.get("by_id_path", device_obj.by_id_path) @@ -821,8 +823,12 @@ def restart_device(device_id, force=False): block_size = (device_obj.aio_block_size or utils.recorded_aio_block_size(snode, device_obj.serial_number) or 0) + # aio, or uring for a device-mapper node. + device_obj.lblk_io_engine = utils.lblk_io_engine_of_device( + snode, device_obj, live, filename) utils.create_aio_bdev(snode.rpc_client(), device_obj.nvme_bdev, - filename, block_size) + filename, block_size, + io_engine=device_obj.lblk_io_engine) snode.rpc_client().bdev_examine(device_obj.nvme_bdev) snode.rpc_client().bdev_wait_for_examine() snode.rpc_client().bdev_set_qd_sampling_period( @@ -1649,10 +1655,12 @@ def new_device_from_failed(device_id): if not device_node.rpc_client().bdev_get(device.nvme_bdev): try: filename = device.by_id_path or device.device_path + live = None try: inventory, _ = device_node.client(timeout=30, retry=1).get_blockdevices() for blk in inventory or []: if blk.get("serial") == device.serial_number: + live = blk filename = blk.get("by_id_path") or blk.get("device_path") break except Exception as e: @@ -1664,8 +1672,13 @@ def new_device_from_failed(device_id): block_size = (device.aio_block_size or utils.recorded_aio_block_size(device_node, device.serial_number) or 0) + # aio, or uring for a device-mapper node; the new device + # record below copies it. + device.lblk_io_engine = utils.lblk_io_engine_of_device( + device_node, device, live, filename) utils.create_aio_bdev(device_node.rpc_client(), device.nvme_bdev, - filename, block_size) + filename, block_size, + io_engine=device.lblk_io_engine) device_node.rpc_client().bdev_examine(device.nvme_bdev) device_node.rpc_client().bdev_wait_for_examine() device_node.rpc_client().bdev_set_qd_sampling_period( @@ -2051,9 +2064,11 @@ def _resize_failed(device_id, dev, msg, set_unavailable): def _reopen_alceml(snode, dev): """Let alceml reopen its AIO bdev at the size the block device has now: - resize_begin -> bdev_aio_rescan -> resize_end. Raises ValueError.""" + resize_begin -> bdev_aio_rescan (bdev_uring_rescan for a uring device) + -> resize_end. Raises ValueError.""" device_id = dev.get_id() rpc_client = snode.rpc_client() + io_engine = utils.lblk_io_engine_of_device(snode, dev) try: rpc_client.alceml_bdev_backend_resize_begin( dev.alceml_bdev, request_timeout=RESIZE_RPC_TIMEOUT_SEC) @@ -2064,14 +2079,14 @@ def _reopen_alceml(snode, dev): rescan_error = None try: - rpc_client.bdev_aio_rescan(dev.nvme_bdev) + utils.rescan_lblk_bdev(rpc_client, dev.nvme_bdev, io_engine) except RPCException as e: rescan_error = e if not _reopen_after_resize(rpc_client, dev): raise _resize_failed(device_id, dev, "alceml resize_end failed", True) if rescan_error is not None: - raise _resize_failed(device_id, dev, f"AIO rescan failed: {rescan_error}", False) + raise _resize_failed(device_id, dev, f"{io_engine} rescan failed: {rescan_error}", False) def _growth_result(grown, factor, devices, errors, reason, shrunk=()): diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index 2dfd5310b5..3ec0b4fa53 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -122,6 +122,13 @@ class NVMeDevice(BaseModel): # Also kept in the node's ``lblk_devices`` entries (see # utils.recorded_aio_block_size). aio_block_size: int = 0 + # I/O engine of the base bdev of an lblk device: "aio" (bdev_aio) or + # "uring" (bdev_uring, for device-mapper nodes such as multipath maps, + # which refuse the RWF_NOWAIT submits of bdev_aio). Only the + # create/delete/rescan RPCs differ; the bdev name and bdev_type "aio" + # are the same. Does not change the on-disk format. Also kept in the + # node's ``lblk_devices`` entries (see utils.recorded_lblk_io_engine). + lblk_io_engine: str = "aio" # The latest disk-health assessment an external assessor (the Kubernetes # operator) recorded through PUT .../devices/{id}/health-assessment, # stored verbatim: {"reporter", "reported_at", "valid_seconds", diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 8648a0b631..f801249f82 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1475,6 +1475,23 @@ def bdev_aio_rescan(self, name): """Re-read the backing device's size (device grow pickup).""" return self._request("bdev_aio_rescan", name=name) + def bdev_uring_create(self, name, filename, block_size=0): + """Create an SPDK io_uring bdev over a Linux block device (lblk + cluster mode, device-mapper nodes: see constants.LBLK_IO_ENGINE_URING). + Same arguments as bdev_aio_create; callers go through + utils.create_aio_bdev with io_engine="uring".""" + kwargs = {"name": name, "filename": filename} + if block_size: + kwargs["block_size"] = block_size + return self._request("bdev_uring_create", **kwargs) + + def bdev_uring_delete(self, name): + return self._request("bdev_uring_delete", name=name) + + def bdev_uring_rescan(self, name): + """Re-read the backing device's size (device grow pickup).""" + return self._request("bdev_uring_rescan", name=name) + def alceml_bdev_backend_resize_begin(self, name, request_timeout=None): """Put an alceml bdev into its resize state: new I/O is held, pending I/O drains, and the backing bdev is closed. Answers once the backing diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index ea35ecedd2..c0fddfcf7b 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -1758,6 +1758,7 @@ def _create_jm_stack_on_device(rpc_client, nvme, snode: StorageNode, after_resta 'nvme_bdev': nvme.nvme_bdev, "serial_number": nvme.serial_number, 'aio_block_size': nvme.aio_block_size, + 'lblk_io_engine': nvme.lblk_io_engine, "device_data_dict": nvme.to_dict(), 'jm_bdev': jm_bdev, 'testing_bdev': test_name, @@ -8511,6 +8512,9 @@ def _reconcile_discovered_devices(snode, nvme_devs): # got it from the node's lblk_devices entry). if found_dev.aio_block_size and not db_dev.aio_block_size: db_dev.aio_block_size = found_dev.aio_block_size + # The I/O engine found_dev was created with (the recorded + # one, or uring for a device-mapper node). + db_dev.lblk_io_engine = found_dev.lblk_io_engine else: db_dev.nvme_controller = found_dev.nvme_controller db_dev.pcie_address = found_dev.pcie_address diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 492e08001a..4ff0e54066 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1617,6 +1617,18 @@ def resolve_lblk_entries(configured_entries, host_devices): # very same device: a different device in the same slot is new. if entry.get("aio_block_size") and resolved_entry["serial"] == entry.get("serial"): resolved_entry["aio_block_size"] = int(entry["aio_block_size"]) + # I/O engine inputs: the kernel name and lsblk type (an agent that + # predates them sends neither; a /dev/mapper/ or /dev/dm- path still + # shows a device-mapper node), the engine the rule gives, and the + # engine recorded on the config entry for the very same device. + if live.get("kname"): + resolved_entry["kname"] = live["kname"] + if live.get("type"): + resolved_entry["dev_type"] = live["type"] + resolved_entry["io_engine"] = lblk_io_engine_for(resolved_entry) + if (entry.get("lblk_io_engine") in constants.LBLK_IO_ENGINES + and resolved_entry["serial"] == entry.get("serial")): + resolved_entry["lblk_io_engine"] = entry["lblk_io_engine"] resolved.append(resolved_entry) return resolved, missing @@ -1716,28 +1728,182 @@ def check_aio_block_size(name, bdev, block_size): f"{block_size}; refusing to use it as storage") +_DM_PATH_PREFIXES = ("/dev/dm-", "/dev/mapper/") +_DM_BY_ID_PREFIXES = ("dm-name-", "dm-uuid-") + + +def _is_dm_path(path) -> bool: + """A device path that names a device-mapper node: /dev/dm-N, + /dev/mapper/, or a /dev/disk/by-id/dm-name-* / dm-uuid-* link.""" + if not isinstance(path, str) or not path: + return False + if path.startswith(_DM_PATH_PREFIXES): + return True + return (path.startswith("/dev/disk/by-id/") + and path.rsplit("/", 1)[-1].startswith(_DM_BY_ID_PREFIXES)) + + +def lblk_io_engine_for(dev) -> str: + """The I/O engine the rule gives a block device (inventory dict, + resolved entry, or a dict of paths): "uring" for a device-mapper node, + "aio" for everything else (see constants.LBLK_IO_ENGINE_URING). + + A device-mapper node is one whose kernel name starts with "dm-" (this + includes a partition kpartx mapped from a multipath map), whose lsblk + TYPE is one of LBLK_DM_TYPES, or, for an agent that predates ``kname`` + and the type, whose path is /dev/dm-*, /dev/mapper/* or a by-id + dm-name-* / dm-uuid-* link. Whole disks, sd*/nvme* partitions and md + arrays stay on aio: md honours RWF_NOWAIT (EAGAIN is retried by SPDK).""" + if not isinstance(dev, dict): + return constants.LBLK_IO_ENGINE_AIO + for key in ("kname", "name"): + if str(dev.get(key) or "").startswith("dm-"): + return constants.LBLK_IO_ENGINE_URING + for key in ("dev_type", "type"): + if dev.get(key) in constants.LBLK_DM_TYPES: + return constants.LBLK_IO_ENGINE_URING + for key in ("path", "filename", "device_path", "current_path", "by_id", "by_id_path"): + if _is_dm_path(dev.get(key)): + return constants.LBLK_IO_ENGINE_URING + return constants.LBLK_IO_ENGINE_AIO + + +def recorded_lblk_io_engine(snode, serial): + """The I/O engine recorded for the device with this serial: its + NVMeDevice record, the node's journal device record and the node's + ``lblk_devices`` entry (see recorded_aio_block_size for why all three). + "uring" when any of them says so (a device is never moved back from + uring implicitly), "aio" when it is recorded otherwise, None when nothing + is recorded.""" + engines = [] + devices = getattr(snode, "nvme_devices", None) + for dev in devices if isinstance(devices, list) else []: + if getattr(dev, "serial_number", None) == serial: + engines.append(getattr(dev, "lblk_io_engine", None)) + jm = getattr(snode, "jm_device", None) + if jm is not None: + data = getattr(jm, "device_data_dict", None) + data = data if isinstance(data, dict) else {} + if data.get("serial_number") == serial or getattr(jm, "serial_number", None) == serial: + engines.append(data.get("lblk_io_engine")) + engines.append(getattr(jm, "lblk_io_engine", None)) + entries = getattr(snode, "lblk_devices", None) + for entry in entries if isinstance(entries, list) else []: + if isinstance(entry, dict) and entry.get("serial") == serial: + engines.append(entry.get("lblk_io_engine")) + engines = [e for e in engines if e in constants.LBLK_IO_ENGINES] + if constants.LBLK_IO_ENGINE_URING in engines: + return constants.LBLK_IO_ENGINE_URING + return constants.LBLK_IO_ENGINE_AIO if engines else None + + +def choose_lblk_io_engine(recorded, derived, what="") -> str: + """The engine to create a device's base bdev with, from the recorded + engine (None: nothing recorded) and the one the rule gives now + (lblk_io_engine_for). The engine does not change the on-disk format, so + it may be re-derived; the recorded one is kept, except that the rule's + uring wins over a recorded aio: bdev_aio cannot serve a device-mapper + node at all (e.g. a device recorded before the engine existed).""" + if derived == constants.LBLK_IO_ENGINE_URING: + if recorded == constants.LBLK_IO_ENGINE_AIO: + logger.warning(f"{what}: recorded I/O engine is aio, but the device is a " + f"device-mapper node, which bdev_aio cannot serve; using uring") + return constants.LBLK_IO_ENGINE_URING + if recorded == constants.LBLK_IO_ENGINE_URING: + return constants.LBLK_IO_ENGINE_URING + return constants.LBLK_IO_ENGINE_AIO + + +def lblk_io_engine_of_device(snode, dev, live=None, filename=None) -> str: + """The engine for (re-)creating the base bdev of an existing lblk device + record ``dev``: its recorded engine (the record itself, the journal + record, the node's config entry), with the rule applied to the live + inventory dict ``live`` (if any), the ``filename`` the bdev is opened on + and the stored paths.""" + serial = getattr(dev, "serial_number", None) + engines = [recorded_lblk_io_engine(snode, serial), getattr(dev, "lblk_io_engine", None)] + engines = [e for e in engines if e in constants.LBLK_IO_ENGINES] + recorded = None + if engines: + recorded = (constants.LBLK_IO_ENGINE_URING + if constants.LBLK_IO_ENGINE_URING in engines + else constants.LBLK_IO_ENGINE_AIO) + probe = {"device_path": getattr(dev, "device_path", "") or "", + "by_id_path": getattr(dev, "by_id_path", "") or ""} + if isinstance(live, dict): + probe.update(live) + probe["filename"] = filename or "" + return choose_lblk_io_engine(recorded, lblk_io_engine_for(probe), + what=f"Device {serial}") + + +def remember_lblk_io_engine(snode, serial, io_engine): + """Store a device's I/O engine on the node's ``lblk_devices`` entry, next + to NVMeDevice.lblk_io_engine (see recorded_lblk_io_engine).""" + entries = getattr(snode, "lblk_devices", None) + if io_engine not in constants.LBLK_IO_ENGINES or not isinstance(entries, list): + return + for entry in entries: + if isinstance(entry, dict) and entry.get("serial") == serial: + entry["lblk_io_engine"] = io_engine + + +def bdev_io_engine(bdev): + """The engine of a live bdev (a bdev_get result) by its driver-specific + info: "aio", "uring", or None when it does not say.""" + driver = bdev.get("driver_specific") if isinstance(bdev, dict) else None + if isinstance(driver, dict): + for engine in (constants.LBLK_IO_ENGINE_URING, constants.LBLK_IO_ENGINE_AIO): + if engine in driver: + return engine + return None + + +def _lblk_bdev_rpc(rpc_client, io_engine, op): + """The bdev_aio_ or bdev_uring_ method of ``rpc_client``.""" + if io_engine == constants.LBLK_IO_ENGINE_URING: + return getattr(rpc_client, f"bdev_uring_{op}"), f"bdev_uring_{op}" + return getattr(rpc_client, f"bdev_aio_{op}"), f"bdev_aio_{op}" + + +def delete_lblk_bdev(rpc_client, name, io_engine=constants.LBLK_IO_ENGINE_AIO): + """bdev_aio_delete or bdev_uring_delete, by the device's I/O engine.""" + delete, _ = _lblk_bdev_rpc(rpc_client, io_engine, "delete") + return delete(name) + + +def rescan_lblk_bdev(rpc_client, name, io_engine=constants.LBLK_IO_ENGINE_AIO): + """bdev_aio_rescan or bdev_uring_rescan (re-read the size of the block + device), by the device's I/O engine.""" + rescan, _ = _lblk_bdev_rpc(rpc_client, io_engine, "rescan") + return rescan(name) + + def create_aio_bdev(rpc_client, name, filename, block_size=0, new_device=False, - verify=True): - """bdev_aio_create with ``block_size`` (0: none, SPDK uses the logical - block size), then (``verify``) check that the bdev has that block size; - a caller that reads the bdev anyway passes verify=False and runs - check_aio_block_size itself. Returns the block size requested. Raises - when the bdev cannot be created or its block size differs from the one - requested: building alceml over a bdev of another block size would make - the device's data unreadable. Only for a ``new_device`` (nothing - formatted on it yet) a refused block size is retried without one.""" + verify=True, io_engine=constants.LBLK_IO_ENGINE_AIO): + """Create the base bdev of an lblk device: bdev_aio_create, or for + ``io_engine`` "uring" bdev_uring_create (same arguments, same block-size + rules), with ``block_size`` (0: none, SPDK uses the logical block size), + then (``verify``) check that the bdev has that block size; a caller that + reads the bdev anyway passes verify=False and runs check_aio_block_size + itself. Returns the block size requested. Raises when the bdev cannot be + created or its block size differs from the one requested: building + alceml over a bdev of another block size would make the device's data + unreadable. Only for a ``new_device`` (nothing formatted on it yet) a + refused block size is retried without one.""" + create, rpc_name = _lblk_bdev_rpc(rpc_client, io_engine, "create") if block_size: - ret = rpc_client.bdev_aio_create(name, filename, block_size=block_size) + ret = create(name, filename, block_size=block_size) if not ret and new_device: - logger.warning(f"bdev_aio_create {name} ({filename}) with block size " + logger.warning(f"{rpc_name} {name} ({filename}) with block size " f"{block_size} failed on {rpc_client.host}; retrying with " f"the device's logical block size") block_size = 0 - ret = rpc_client.bdev_aio_create(name, filename) + ret = create(name, filename) else: - ret = rpc_client.bdev_aio_create(name, filename) + ret = create(name, filename) if not ret: - raise Exception(f"bdev_aio_create failed for {name} ({filename}) on {rpc_client.host}") + raise Exception(f"{rpc_name} failed for {name} ({filename}) on {rpc_client.host}") if verify: check_aio_block_size(name, rpc_client.bdev_get(name), block_size) return block_size @@ -1758,6 +1924,14 @@ def addAioDevices(rpc_client, snode, blk_entries): recorded = recorded_aio_block_size(snode, entry["serial"]) if recorded is None and entry.get("aio_block_size"): recorded = int(entry["aio_block_size"]) + # The I/O engine: the recorded one, unless the rule says uring (a + # device-mapper node, which bdev_aio cannot serve). + recorded_engine = recorded_lblk_io_engine(snode, entry["serial"]) + if recorded_engine is None and entry.get("lblk_io_engine") in constants.LBLK_IO_ENGINES: + recorded_engine = entry["lblk_io_engine"] + io_engine = choose_lblk_io_engine( + recorded_engine, entry.get("io_engine") or lblk_io_engine_for(entry), + what=f"Block device {entry.get('name')} ({bdev_name})") ret = rpc_client.bdev_get(bdev_name) if not ret: # A path the caller selected the device by is opened as given. @@ -1767,12 +1941,20 @@ def addAioDevices(rpc_client, snode, blk_entries): if recorded is None: expected = create_aio_bdev(rpc_client, bdev_name, filename, choose_aio_block_size(entry), - new_device=True, verify=False) + new_device=True, verify=False, + io_engine=io_engine) else: expected = create_aio_bdev(rpc_client, bdev_name, filename, - recorded, verify=False) + recorded, verify=False, + io_engine=io_engine) else: expected = recorded + # A bdev that is still there is used as it is; record its engine. + live_engine = bdev_io_engine(ret) + if live_engine and live_engine != io_engine: + logger.warning(f"Base bdev {bdev_name} exists as a {live_engine} bdev, " + f"expected {io_engine}; using it as it is") + io_engine = live_engine rpc_client.bdev_examine(bdev_name) rpc_client.bdev_wait_for_examine() @@ -1787,6 +1969,7 @@ def addAioDevices(rpc_client, snode, blk_entries): # A new device is formatted at the block size its bdev has now. aio_block_size = int(bdev['block_size']) if recorded is None else recorded remember_aio_block_size(snode, entry["serial"], aio_block_size) + remember_lblk_io_engine(snode, entry["serial"], io_engine) # Queue-depth sampling feeds the control-plane hung-IO watchdog # (AIO has no bdev_nvme-style timeout/action_on_timeout). @@ -1809,6 +1992,7 @@ def addAioDevices(rpc_client, snode, blk_entries): 'nvme_controller': "", 'bdev_type': "aio", 'aio_block_size': aio_block_size, + 'lblk_io_engine': io_engine, 'device_path': entry["current_path"], 'by_id_path': entry.get("path") or entry.get("by_id", ""), 'node_id': snode.get_id(), diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index 07dbc7bccc..3882a13caa 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -317,7 +317,7 @@ def get_block_devices_info() -> list[dict]: logger.debug("function:get_block_devices_info start") out, err, rc = shell_utils.run_command( - "lsblk -J -b -o NAME,TYPE,SIZE,SERIAL,WWN,MOUNTPOINT,MODEL,ROTA,RO,VENDOR,PKNAME,PARTUUID") + "lsblk -J -b -o NAME,KNAME,TYPE,SIZE,SERIAL,WWN,MOUNTPOINT,MODEL,ROTA,RO,VENDOR,PKNAME,PARTUUID") if rc != 0: logger.error("Error running lsblk: %s", err) return [] @@ -356,6 +356,10 @@ def get_block_devices_info() -> list[dict]: physical_bs = _physical_block_size(name) devices.append({ "name": name, + # Kernel name (dm-N for a device-mapper node, whose NAME is the + # map name) and TYPE: the control plane serves device-mapper + # nodes through bdev_uring (utils.lblk_io_engine_for). + "kname": dev.get("kname") or name, "device_path": f"/dev/{name}", "type": dev.get("type"), "size": int(dev.get("size") or 0), @@ -391,6 +395,7 @@ def get_block_devices_info() -> list[dict]: part_synthetic = True devices.append({ "name": part_name, + "kname": child.get("kname") or part_name, "device_path": f"/dev/{part_name}", "type": "part", "size": int(child.get("size") or 0), diff --git a/tests/unit/test_lblk_io_engine.py b/tests/unit/test_lblk_io_engine.py new file mode 100644 index 0000000000..303261cfc6 --- /dev/null +++ b/tests/unit/test_lblk_io_engine.py @@ -0,0 +1,470 @@ +"""Unit tests for the I/O engine of lblk base bdevs. + +SPDK's bdev_aio submits with RWF_NOWAIT, which device-mapper honours only +when every target does; dm-multipath does not, so a multipath map cannot be +served by bdev_aio at all. Device-mapper nodes are served by bdev_uring +instead, everything else keeps bdev_aio. The bdev name (aio_) and +NVMeDevice.bdev_type "aio" are the same for both engines. + +Covered: + - utils.lblk_io_engine_for: dm-* kernel name, lsblk TYPE mpath/lvm/crypt/dm, + a kpartx partition of a multipath map, /dev/mapper and by-id dm-* paths + (older agent), sd/nvme disks and partitions and md arrays -> aio. + - utils.resolve_lblk_entries carries kname/type, the derived engine and the + recorded engine (same device only). + - utils.addAioDevices: dm device -> bdev_uring_create with the block size, + recorded on the device and the node's lblk_devices entry; sd/md -> aio; + recorded uring kept; recorded aio on a dm node -> uring; older agent; + an existing bdev's engine is kept. + - delete / rescan route by engine; device growth rescans a uring device + with bdev_uring_rescan. + - storage_node_ops._reconcile_discovered_devices takes the engine over. + - device_controller.restart_device / new_device_from_failed create the + bdev with the device's engine. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import constants, utils +from simplyblock_core.controllers import device_controller +from simplyblock_core.models.nvme_device import JMDevice, NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.storage_node_ops import _reconcile_discovered_devices + +GIB = 1 << 30 +AIO = constants.LBLK_IO_ENGINE_AIO +URING = constants.LBLK_IO_ENGINE_URING + + +def _entry(serial="S1", name="sdb", current_path="/dev/sdb", by_id=None, **extra): + e = {"name": name, "serial": serial, + "by_id": by_id if by_id is not None else f"/dev/disk/by-id/wwn-{serial}", + "size": 100 * GIB, "numa": 0, "model": "M", "current_path": current_path, + "has_partitions": False, "logical_block_size": 512, + "physical_block_size": 4096} + e.update(extra) + return e + + +def _mpath_entry(serial="S1", **extra): + return _entry(serial=serial, name="mpatha", current_path="/dev/mpatha", + by_id=f"/dev/disk/by-id/dm-uuid-mpath-{serial}", + kname="dm-0", dev_type="mpath", **extra) + + +def _snode(devices=(), lblk_devices=None, jm_device=None): + n = StorageNode() + n.uuid = "node-1" + n.cluster_id = "cluster-1" + n.nvme_devices = list(devices) + n.lblk_devices = lblk_devices if lblk_devices is not None else [ + {"name": "sdb", "serial": "S1", "size": 100 * GIB}] + if jm_device is not None: + n.jm_device = jm_device + return n + + +def _record(serial="S1", engine=AIO, aio_block_size=4096, + status=NVMeDevice.STATUS_ONLINE): + d = NVMeDevice() + d.uuid = f"dev-{serial}" + d.serial_number = serial + d.bdev_type = "aio" + d.nvme_bdev = utils.aio_bdev_name_for_serial(serial) + d.aio_block_size = aio_block_size + d.lblk_io_engine = engine + d.status = status + return d + + +class _FakeRpc: + """SPDK fake with both base-bdev engines; records every create.""" + + def __init__(self, existing=None): + self.host = "test-host" + self.bdevs = dict(existing or {}) + self.creates = [] # (engine, name, filename, block_size) + + def bdev_get(self, name): + return self.bdevs.get(name) + + def _create(self, engine, name, filename, block_size): + self.creates.append((engine, name, filename, block_size)) + bs = block_size or 512 + self.bdevs[name] = {"name": name, "block_size": bs, + "num_blocks": 100 * GIB // bs, + "driver_specific": {engine: {"filename": filename}}} + return name + + def bdev_aio_create(self, name, filename, block_size=0): + return self._create(AIO, name, filename, block_size) + + def bdev_uring_create(self, name, filename, block_size=0): + return self._create(URING, name, filename, block_size) + + def bdev_examine(self, name): + return True + + def bdev_wait_for_examine(self): + return True + + def bdev_set_qd_sampling_period(self, name, period): + return True + + +class TestEngineRule(unittest.TestCase): + + def test_dm_kernel_name_is_uring(self): + self.assertEqual(utils.lblk_io_engine_for({"name": "mpatha", "kname": "dm-3"}), URING) + + def test_dm_types_are_uring(self): + for t in ("mpath", "lvm", "crypt", "dm"): + with self.subTest(t=t): + self.assertEqual(utils.lblk_io_engine_for({"name": "x", "type": t}), URING) + + def test_kpartx_partition_of_multipath_is_uring(self): + self.assertEqual(utils.lblk_io_engine_for( + {"name": "mpatha1", "kname": "dm-4", "type": "part"}), URING) + + def test_mapper_and_dm_paths_are_uring(self): + for path in ("/dev/mapper/mpatha", "/dev/dm-0", + "/dev/disk/by-id/dm-uuid-mpath-3600a", "/dev/disk/by-id/dm-name-mpatha"): + with self.subTest(path=path): + self.assertEqual(utils.lblk_io_engine_for( + {"name": "x", "device_path": path}), URING) + + def test_plain_devices_are_aio(self): + for dev in ({"name": "sdb", "kname": "sdb", "type": "disk", + "device_path": "/dev/sdb", "by_id_path": "/dev/disk/by-id/wwn-0x5"}, + {"name": "nvme0n1p2", "kname": "nvme0n1p2", "type": "part"}, + {"name": "sdc1", "type": "part", "device_path": "/dev/sdc1"}, + {"name": "md0", "kname": "md0", "type": "raid5"}, + {"name": "md127", "type": "raid1", "device_path": "/dev/md127"}): + with self.subTest(dev=dev["name"]): + self.assertEqual(utils.lblk_io_engine_for(dev), AIO) + + def test_older_agent_without_fields_is_aio(self): + self.assertEqual(utils.lblk_io_engine_for( + {"name": "sdb", "device_path": "/dev/sdb", "by_id_path": ""}), AIO) + + def test_choose_prefers_recorded_unless_rule_says_uring(self): + self.assertEqual(utils.choose_lblk_io_engine(None, AIO), AIO) + self.assertEqual(utils.choose_lblk_io_engine(URING, AIO), URING) + self.assertEqual(utils.choose_lblk_io_engine(AIO, URING), URING) + self.assertEqual(utils.choose_lblk_io_engine(AIO, AIO), AIO) + + +class TestResolveCarriesEngine(unittest.TestCase): + + def _live(self, serial, **extra): + d = {"name": "mpatha", "kname": "dm-0", "type": "mpath", + "device_path": "/dev/mpatha", "serial": serial, "by_id_path": "", + "size": 100 * GIB, "numa_node": 0} + d.update(extra) + return d + + def test_dm_device_resolves_to_uring(self): + resolved, _ = utils.resolve_lblk_entries( + [{"name": "mpatha", "serial": "S1", "size": 1}], [self._live("S1")]) + self.assertEqual(resolved[0]["kname"], "dm-0") + self.assertEqual(resolved[0]["dev_type"], "mpath") + self.assertEqual(resolved[0]["io_engine"], URING) + + def test_disk_resolves_to_aio(self): + live = self._live("S1", name="sdb", kname="sdb", type="disk", device_path="/dev/sdb") + resolved, _ = utils.resolve_lblk_entries( + [{"name": "sdb", "serial": "S1", "size": 1}], [live]) + self.assertEqual(resolved[0]["io_engine"], AIO) + + def test_older_agent_mapper_path(self): + live = {"name": "mpatha", "device_path": "/dev/mapper/mpatha", "serial": "S1", + "by_id_path": "", "size": 100 * GIB, "numa_node": 0} + resolved, _ = utils.resolve_lblk_entries( + [{"name": "mpatha", "serial": "S1", "size": 1}], [live]) + self.assertEqual(resolved[0]["io_engine"], URING) + + def test_recorded_engine_carried_for_same_device_only(self): + live = self._live("S1", name="sdb", kname="sdb", type="disk", device_path="/dev/sdb") + resolved, _ = utils.resolve_lblk_entries( + [{"name": "sdb", "serial": "S1", "size": 1, "lblk_io_engine": URING}], [live]) + self.assertEqual(resolved[0]["lblk_io_engine"], URING) + live = dict(live, serial="OTHER") + resolved, _ = utils.resolve_lblk_entries( + [{"name": "sdb", "serial": "S1", "size": 1, "lblk_io_engine": URING}], [live]) + self.assertNotIn("lblk_io_engine", resolved[0]) + + +class TestAddAioDevicesEngine(unittest.TestCase): + + def test_new_multipath_device_created_with_uring(self): + rpc = _FakeRpc() + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_mpath_entry(io_engine=URING)]) + self.assertEqual(rpc.creates, [(URING, "aio_S1", "/dev/disk/by-id/dm-uuid-mpath-S1", 4096)]) + self.assertEqual(devs[0].lblk_io_engine, URING) + self.assertEqual(devs[0].nvme_bdev, "aio_S1") + self.assertEqual(devs[0].bdev_type, "aio") + self.assertEqual(devs[0].aio_block_size, 4096) + self.assertEqual(snode.lblk_devices[0]["lblk_io_engine"], URING) + + def test_new_disk_created_with_aio(self): + rpc = _FakeRpc() + snode = _snode() + devs = utils.addAioDevices(rpc, snode, [_entry(io_engine=AIO)]) + self.assertEqual(rpc.creates[0][0], AIO) + self.assertEqual(devs[0].lblk_io_engine, AIO) + self.assertEqual(snode.lblk_devices[0]["lblk_io_engine"], AIO) + + def test_md_array_created_with_aio(self): + rpc = _FakeRpc() + entry = _entry(name="md0", current_path="/dev/md0", by_id="/dev/disk/by-id/md-uuid-1", + kname="md0", dev_type="raid5") + devs = utils.addAioDevices(rpc, _snode(), [entry]) + self.assertEqual(rpc.creates[0][0], AIO) + self.assertEqual(devs[0].lblk_io_engine, AIO) + + def test_entry_without_derived_engine_uses_rule(self): + rpc = _FakeRpc() + entry = _mpath_entry() + utils.addAioDevices(rpc, _snode(), [entry]) + self.assertEqual(rpc.creates[0][0], URING) + + def test_older_agent_disk_without_fields_is_aio(self): + rpc = _FakeRpc() + entry = _entry() + entry.pop("logical_block_size") + utils.addAioDevices(rpc, _snode(), [entry]) + self.assertEqual(rpc.creates[0][0], AIO) + + def test_recorded_uring_kept(self): + # e.g. an older agent that reports the map under a by-id wwn link only + rpc = _FakeRpc() + snode = _snode(devices=[_record(engine=URING)]) + devs = utils.addAioDevices(rpc, snode, [_entry(io_engine=AIO)]) + self.assertEqual(rpc.creates, [(URING, "aio_S1", "/dev/disk/by-id/wwn-S1", 4096)]) + self.assertEqual(devs[0].lblk_io_engine, URING) + + def test_recorded_uring_on_lblk_entry_kept(self): + rpc = _FakeRpc() + snode = _snode(devices=[_record(engine=AIO)], + lblk_devices=[{"name": "sdb", "serial": "S1", "lblk_io_engine": URING}]) + utils.addAioDevices(rpc, snode, [_entry()]) + self.assertEqual(rpc.creates[0][0], URING) + + def test_recorded_aio_on_dm_node_switches_to_uring(self): + rpc = _FakeRpc() + snode = _snode(devices=[_record(engine=AIO, aio_block_size=512)]) + with self.assertLogs(utils.logger, level="WARNING") as logs: + devs = utils.addAioDevices(rpc, snode, [_mpath_entry(io_engine=URING)]) + self.assertEqual(rpc.creates[0][0], URING) + self.assertEqual(rpc.creates[0][3], 512) # the recorded block size is kept + self.assertEqual(devs[0].lblk_io_engine, URING) + self.assertTrue(any("using uring" in m for m in logs.output)) + + def test_journal_record_engine_used(self): + jm = JMDevice() + jm.serial_number = "S1" + jm.device_data_dict = {"serial_number": "S1", "lblk_io_engine": URING, + "aio_block_size": 4096} + rpc = _FakeRpc() + utils.addAioDevices(rpc, _snode(jm_device=jm), [_entry()]) + self.assertEqual(rpc.creates[0][0], URING) + + def test_existing_bdev_engine_recorded(self): + existing = {"aio_S1": {"name": "aio_S1", "block_size": 4096, + "num_blocks": 100 * GIB // 4096, + "driver_specific": {URING: {"filename": "/dev/dm-0"}}}} + rpc = _FakeRpc(existing=existing) + devs = utils.addAioDevices(rpc, _snode(devices=[_record(engine=URING)]), [_entry()]) + self.assertEqual(rpc.creates, []) + self.assertEqual(devs[0].lblk_io_engine, URING) + + +class TestDeleteRescanRouting(unittest.TestCase): + + def test_delete_routes_by_engine(self): + rpc = MagicMock() + utils.delete_lblk_bdev(rpc, "aio_S1", URING) + rpc.bdev_uring_delete.assert_called_once_with("aio_S1") + rpc.bdev_aio_delete.assert_not_called() + rpc = MagicMock() + utils.delete_lblk_bdev(rpc, "aio_S1", AIO) + rpc.bdev_aio_delete.assert_called_once_with("aio_S1") + rpc.bdev_uring_delete.assert_not_called() + + def test_rescan_routes_by_engine(self): + rpc = MagicMock() + utils.rescan_lblk_bdev(rpc, "aio_S1", URING) + rpc.bdev_uring_rescan.assert_called_once_with("aio_S1") + rpc.bdev_aio_rescan.assert_not_called() + rpc = MagicMock() + utils.rescan_lblk_bdev(rpc, "aio_S1") + rpc.bdev_aio_rescan.assert_called_once_with("aio_S1") + + def test_create_routes_by_engine_with_block_size(self): + rpc = _FakeRpc() + self.assertEqual(utils.create_aio_bdev(rpc, "aio_S1", "/dev/dm-0", 4096, + io_engine=URING), 4096) + self.assertEqual(rpc.creates, [(URING, "aio_S1", "/dev/dm-0", 4096)]) + + def test_reopen_alceml_rescans_uring_device(self): + dev = _record(engine=URING) + dev.alceml_bdev = "alceml_1" + snode = _snode(devices=[dev]) + rpc = MagicMock() + with patch.object(StorageNode, "rpc_client", return_value=rpc): + device_controller._reopen_alceml(snode, dev) + rpc.bdev_uring_rescan.assert_called_once_with("aio_S1") + rpc.bdev_aio_rescan.assert_not_called() + + def test_reopen_alceml_rescans_aio_device(self): + dev = _record(engine=AIO) + dev.alceml_bdev = "alceml_1" + snode = _snode(devices=[dev]) + rpc = MagicMock() + with patch.object(StorageNode, "rpc_client", return_value=rpc): + device_controller._reopen_alceml(snode, dev) + rpc.bdev_aio_rescan.assert_called_once_with("aio_S1") + rpc.bdev_uring_rescan.assert_not_called() + + +class TestReconcileTakesEngine(unittest.TestCase): + + def test_engine_taken_over(self): + db_dev = _record(engine=AIO) + found = _record(engine=URING) + _reconcile_discovered_devices(_snode(devices=[db_dev]), [found]) + self.assertEqual(db_dev.lblk_io_engine, URING) + + +class TestDeviceControllerEngine(unittest.TestCase): + + def _restart(self, device, inventory=(), lblk_devices=None): + device.node_id = "node-1" + device.cluster_id = "cluster-1" + device.nvmf_nqn = "" + device.alceml_bdev = "" + snode = MagicMock() + snode.cluster_id = "cluster-1" + snode.nvme_devices = [device] + snode.jm_device = None + snode.lblk_devices = lblk_devices or [] + + rpc = MagicMock() + state = {"present": False} + rpc.bdev_get.side_effect = ( + lambda name: {"name": name, "block_size": 4096} if state["present"] else None) + + def _create(name, filename, block_size=0): + state["present"] = True + return name + + rpc.bdev_aio_create.side_effect = _create + rpc.bdev_uring_create.side_effect = _create + snode.rpc_client.return_value = rpc + snode.client.return_value.get_blockdevices.return_value = (list(inventory), None) + + db = MagicMock() + db.get_storage_device_by_id.return_value = device + db.get_storage_node_by_id.return_value = snode + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(device_controller.tasks_controller, + "get_active_dev_restart_task", return_value=None), \ + patch.object(device_controller, "device_set_retries_exhausted"), \ + patch.object(device_controller, "device_set_unavailable"), \ + patch.object(device_controller, "_def_create_device_stack", return_value=True), \ + patch.object(device_controller, "device_set_io_error"), \ + patch.object(device_controller, "device_set_online"), \ + patch.object(device_controller, "device_events"): + result = device_controller.restart_device(device.uuid) + return result, rpc + + def test_restart_uring_device(self): + device = _record(engine=URING, status=NVMeDevice.STATUS_REMOVED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + result, rpc = self._restart(device) + self.assertTrue(result) + rpc.bdev_uring_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + rpc.bdev_aio_create.assert_not_called() + + def test_restart_aio_device(self): + device = _record(engine=AIO, status=NVMeDevice.STATUS_REMOVED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + result, rpc = self._restart(device) + self.assertTrue(result) + rpc.bdev_aio_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + rpc.bdev_uring_create.assert_not_called() + + def test_restart_recorded_aio_dm_device_gets_uring(self): + device = _record(engine=AIO, status=NVMeDevice.STATUS_REMOVED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + inventory = [{"serial": "S1", "name": "mpatha", "kname": "dm-0", "type": "mpath", + "device_path": "/dev/mpatha", + "by_id_path": "/dev/disk/by-id/dm-uuid-mpath-S1"}] + result, rpc = self._restart(device, inventory=inventory) + self.assertTrue(result) + rpc.bdev_uring_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/dm-uuid-mpath-S1", block_size=4096) + self.assertEqual(device.lblk_io_engine, URING) + + def test_restart_engine_from_lblk_entry(self): + device = _record(engine=AIO, status=NVMeDevice.STATUS_REMOVED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + result, rpc = self._restart( + device, lblk_devices=[{"serial": "S1", "lblk_io_engine": URING}]) + self.assertTrue(result) + rpc.bdev_uring_create.assert_called_once() + + def test_new_device_from_failed_keeps_engine(self): + device = _record(engine=URING, status=NVMeDevice.STATUS_FAILED_AND_MIGRATED) + device.by_id_path = "/dev/disk/by-id/wwn-1" + node = MagicMock() + node.get_id.return_value = "node-1" + node.nvme_devices = [device] + node.lblk_devices = [] + rpc = MagicMock() + state = {"present": False} + rpc.bdev_get.side_effect = ( + lambda name: {"name": name, "block_size": 4096} if state["present"] else None) + + def _create(name, filename, block_size=0): + state["present"] = True + return name + + rpc.bdev_uring_create.side_effect = _create + node.rpc_client.return_value = rpc + node.client.return_value.get_blockdevices.return_value = ([], None) + db = MagicMock() + db.get_storage_node_by_device_id.return_value = node + with patch.object(device_controller, "DBController", return_value=db): + self.assertTrue(device_controller.new_device_from_failed(device.uuid)) + rpc.bdev_uring_create.assert_called_once_with( + "aio_S1", "/dev/disk/by-id/wwn-1", block_size=4096) + rpc.bdev_aio_create.assert_not_called() + + +class TestNoDirectAioRpcCalls(unittest.TestCase): + """Every bdev_aio_create/delete/rescan goes through the engine-aware + helpers in utils.""" + + def test_no_direct_calls_outside_helpers(self): + import pathlib + import re + root = pathlib.Path(utils.__file__).resolve().parents[2] + pattern = re.compile(r"\.bdev_aio_(create|delete|rescan)\(") + offenders = [] + for sub in ("simplyblock_core", "simplyblock_web", "simplyblock_cli"): + for path in (root / sub).rglob("*.py"): + if path.name == "rpc_client.py": + continue + for n, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if pattern.search(line): + offenders.append(f"{path.relative_to(root)}:{n}") + self.assertEqual(offenders, []) + + +if __name__ == "__main__": + unittest.main() From 800f763da960f5811c750e0587e80542c6d1ab8c Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 12:06:59 +0100 Subject: [PATCH 07/50] feat(multi-attach): array LUN mode, phase 2 (options, device identity, paths) A cluster whose data devices are protected array LUNs presented to several nodes runs 1+0; an owner's LUN is reopened on another node on failure (phase 3). This adds everything below the takeover itself. Every new path is gated on Cluster.multi_attach / NVMeDevice.multi_attach; regular clusters are unchanged. - cluster create/add --multi-attach and --ftt 1|2 (CLI, v1/v2 API, Cluster.multi_attach): requires --raid-devices, ha_type 'ha', not single-node; forces 1+0 and max_fault_tolerance = ftt. --ftt alone is refused; the npcs>=2 rule for FTT2 does not apply to multi-attach. - floors: nodes >= FTT+1 (>= 3 for FTT2), >= 1 data device per node (one LUN per node recommended), no single/two-node override. - NVMeDevice.multi_attach / lun_wwn / serving_node_id / node_paths; lblk devices of a multi-attach cluster are flagged at add-node / restart (WWN from the lsblk inventory, else the serial). - Device subsystem NQN :lun:; namespace UUID derived from the device UUID (uuid5) -- the device UUID itself is the alceml bdev's UUID alias and would collide with the owner's loopback bdev; device_controller now adds the namespace with that UUID too. - Per-node controller-ID ranges: StorageNode.cntlid_block claimed in one FDB transaction (DBController.claim_cntlid_block), range [i*64+1, i*64+64], at most 1022 nodes; subsystem_create max_cntlid. - Peer controllers to a multi-attach device: multipath mode and per-controller hold (ctrlr_loss/fast_io_fail 120 s, reconnect 2 s). - alceml stuck_io_timeout_ms (60 s, ultra feat/alceml-stuck-io-timeout) for multi-attach devices only; sent only when non-zero. - Owner loopback: the owner's distribs map its own multi-attach devices as remote_alceml_n1; _connect_to_remote_devs, the coverage gate, sync_remote_devices_from_spdk and reconnect_dropped_remote_devs include them. Co-Authored-By: Claude Opus 5.5 --- simplyblock_cli/cli-reference.yaml | 38 ++ simplyblock_cli/cli.py | 4 + simplyblock_cli/clibase.py | 8 + simplyblock_core/cluster_ops.py | 61 +- simplyblock_core/constants.py | 23 + .../controllers/device_controller.py | 17 +- simplyblock_core/db_controller.py | 58 ++ simplyblock_core/distr_controller.py | 13 +- simplyblock_core/floors.py | 23 +- simplyblock_core/models/cluster.py | 7 + simplyblock_core/models/nvme_device.py | 20 + simplyblock_core/models/storage_node.py | 4 + simplyblock_core/multi_attach.py | 166 +++++ simplyblock_core/rpc_client.py | 14 +- simplyblock_core/storage_node_ops.py | 87 ++- simplyblock_core/utils/__init__.py | 3 + simplyblock_web/api/v1/cluster.py | 4 + simplyblock_web/api/v2/_dtos.py | 2 + simplyblock_web/api/v2/cluster/__init__.py | 5 + tests/unit/test_multi_attach.py | 621 ++++++++++++++++++ 20 files changed, 1149 insertions(+), 29 deletions(-) create mode 100644 simplyblock_core/multi_attach.py create mode 100644 tests/unit/test_multi_attach.py diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index d29a4f7627..684fe06c75 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1046,6 +1046,25 @@ commands: dest: raid_devices type: bool action: store_true + - name: "--multi-attach" + help: > + Multi-attach array LUN mode. Every data device is a LUN of an external array that already protects it + and presents it to several storage nodes. The cluster runs 1+0 erasure coding (the data-chunks and + parity-chunks settings are overridden); on a node failure the LUN is reopened on a standby node and + reached over an additional NVMe-oF path. Requires --raid-devices, --ftt and --ha-type ha. Cannot be + changed after cluster creation. + dest: multi_attach + type: bool + action: store_true + - name: "--ftt" + help: > + Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least + 3 storage nodes). Required with --multi-attach, refused without it. + dest: ftt + type: int + choices: + - 1 + - 2 - name: "--enable-failure-domain" help: "Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed." dest: enable_failure_domain @@ -1323,6 +1342,25 @@ commands: dest: raid_devices type: bool action: store_true + - name: "--multi-attach" + help: > + Multi-attach array LUN mode. Every data device is a LUN of an external array that already protects it + and presents it to several storage nodes. The cluster runs 1+0 erasure coding (the data-chunks and + parity-chunks settings are overridden); on a node failure the LUN is reopened on a standby node and + reached over an additional NVMe-oF path. Requires --raid-devices, --ftt and --ha-type ha. Cannot be + changed after cluster creation. + dest: multi_attach + type: bool + action: store_true + - name: "--ftt" + help: > + Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least + 3 storage nodes). Required with --multi-attach, refused without it. + dest: ftt + type: int + choices: + - 1 + - 2 - name: "--enable-failure-domain" help: "Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed." dest: enable_failure_domain diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index dd84160844..33a97664bb 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -468,6 +468,8 @@ def init_cluster__create(self, subparser): subcommand.add_argument('--disable-monitoring', help='Disable monitoring stack, false by default. Default: `false`.', dest='disable_monitoring', action='store_true') subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ nodes); the flag only affects status reporting.', dest='strict_node_anti_affinity', action='store_true') subcommand.add_argument('--raid-devices', help='Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no other device is writable. Cannot be changed after cluster creation.', dest='raid_devices', action='store_true') + subcommand.add_argument('--multi-attach', help='Multi-attach array LUN mode. Every data device is a LUN of an external array that already protects it and presents it to several storage nodes. The cluster runs 1+0 erasure coding (the data-chunks and parity-chunks settings are overridden); on a node failure the LUN is reopened on a standby node and reached over an additional NVMe-oF path. Requires --raid-devices, --ftt and --ha-type ha. Cannot be changed after cluster creation.', dest='multi_attach', action='store_true') + subcommand.add_argument('--ftt', help='Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least 3 storage nodes). Required with --multi-attach, refused without it.', type=int, dest='ftt', choices=[1,2,]) subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') subcommand.add_argument('--sync-replication', help='Enable site-level synchronous replication over exactly two sites. Each storage node must then be added with --site; every lvstore is replicated to the other site. Requires --ha-type ha. Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='sync_replication', action='store_true') subcommand.add_argument('--device-mode', help='Storage-device mode for the whole cluster. \'nvme\' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. \'lblk\': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at \'sn configure\' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected.', type=str, default='nvme', dest='device_mode', choices=['nvme','lblk',]) @@ -516,6 +518,8 @@ def init_cluster__add(self, subparser): subcommand.add_argument('--inflight-io-threshold', help='The number of inflight IOs allowed before the IO queuing starts. Default: `4`.', type=int, default=4, dest='inflight_io_threshold') subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ nodes); the flag only affects status reporting.', dest='strict_node_anti_affinity', action='store_true') subcommand.add_argument('--raid-devices', help='Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no other device is writable. Cannot be changed after cluster creation.', dest='raid_devices', action='store_true') + subcommand.add_argument('--multi-attach', help='Multi-attach array LUN mode. Every data device is a LUN of an external array that already protects it and presents it to several storage nodes. The cluster runs 1+0 erasure coding (the data-chunks and parity-chunks settings are overridden); on a node failure the LUN is reopened on a standby node and reached over an additional NVMe-oF path. Requires --raid-devices, --ftt and --ha-type ha. Cannot be changed after cluster creation.', dest='multi_attach', action='store_true') + subcommand.add_argument('--ftt', help='Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least 3 storage nodes). Required with --multi-attach, refused without it.', type=int, dest='ftt', choices=[1,2,]) subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') subcommand.add_argument('--sync-replication', help='Enable site-level synchronous replication over exactly two sites. Each storage node must then be added with --site; every lvstore is replicated to the other site. Requires --ha-type ha. Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='sync_replication', action='store_true') subcommand.add_argument('--device-mode', help='Storage-device mode for the whole cluster. \'nvme\' (default): NVMe PCIe devices auto-detected and attached via the SPDK nvme driver. \'lblk\': arbitrary Linux block devices wrapped in SPDK AIO bdevs; devices are selected at \'sn configure\' by name or serial number. Deploy-time only; inter-node fabric (nvme-tcp/rdma) is unaffected.', type=str, default='nvme', dest='device_mode', choices=['nvme','lblk',]) diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 73b3d9d8d5..dae719fb56 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -1938,6 +1938,8 @@ def cluster_add(self, args): inflight_io_threshold = args.inflight_io_threshold strict_node_anti_affinity = args.strict_node_anti_affinity raid_devices = getattr(args, 'raid_devices', False) + multi_attach = getattr(args, 'multi_attach', False) + ftt = getattr(args, 'ftt', None) is_single_node = args.is_single_node client_data_nic = args.client_data_nic enable_failure_domain = getattr(args, 'enable_failure_domain', False) @@ -1968,6 +1970,8 @@ def cluster_add(self, args): page_copy=getattr(args, 'page_copy', False), md_journal=md_journal, raid_devices=raid_devices, + multi_attach=multi_attach, + ftt=ftt, ) def cluster_create(self, args): @@ -1998,6 +2002,8 @@ def cluster_create(self, args): disable_monitoring = args.disable_monitoring strict_node_anti_affinity = args.strict_node_anti_affinity raid_devices = getattr(args, 'raid_devices', False) + multi_attach = getattr(args, 'multi_attach', False) + ftt = getattr(args, 'ftt', None) name = args.name tls_secret = args.tls_secret ingress_host_source = args.ingress_host_source @@ -2047,6 +2053,8 @@ def cluster_create(self, args): page_copy=getattr(args, 'page_copy', False), md_journal=md_journal, raid_devices=raid_devices, + multi_attach=multi_attach, + ftt=ftt, ) def query_yes_no(self, question, default="yes"): diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 53593fca09..49fbb2be85 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -378,6 +378,43 @@ def _validate_leader_site(sync_replication, leader_site) -> None: if leader_site and not sync_replication: raise SyncReplicationSiteError( "leader_site is only allowed together with sync_replication") +#: Erasure-coding scheme a multi-attach (array LUN) cluster runs: the array +#: already protects every LUN, so simplyblock adds no parity. +MULTI_ATTACH_SCHEME = (1, 0) + + +def _resolve_multi_attach(multi_attach, ftt, raid_devices, ha_type, is_single_node, + distr_ndcs, distr_npcs, max_fault_tolerance): + """Validate the multi-attach array LUN options and return the effective + ``(distr_ndcs, distr_npcs, max_fault_tolerance)``. + + Without ``multi_attach`` the inputs are returned unchanged (``ftt`` must + then be unset). With it the cluster runs 1+0 (the array protects the + LUNs) and tolerates ``ftt`` node failures by reopening an owner's LUN on + another node, so it needs ``raid_devices``, an HA cluster (the LUN is + attached to the LVS secondary/tertiary) and more than one node. + + Raises: + ValueError: an inconsistent combination of options. + """ + if not multi_attach: + if ftt is not None: + raise ValueError("ftt is only valid together with multi_attach") + return distr_ndcs, distr_npcs, max_fault_tolerance + if not raid_devices: + raise ValueError("multi_attach requires raid_devices: the array must protect every LUN") + if ftt not in (1, 2): + raise ValueError("multi_attach requires ftt 1 or 2") + if ha_type != "ha": + raise ValueError("multi_attach requires ha_type='ha'") + if is_single_node: + raise ValueError("multi_attach cannot be used on a single-node cluster") + if (distr_ndcs, distr_npcs) != MULTI_ATTACH_SCHEME: + logger.info("multi_attach: erasure coding %s+%s overridden by %s+%s", + distr_ndcs, distr_npcs, *MULTI_ATTACH_SCHEME) + return MULTI_ATTACH_SCHEME[0], MULTI_ATTACH_SCHEME[1], int(ftt) + + def _validate_integrity_flags(inline_checksum, atomic_4k): """atomic_4k only lifts the data plane's 4K gate on fallback-mode checksums, so without inline_checksum it is a setting that does nothing.""" @@ -405,7 +442,12 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, page_copy=False, md_journal=True, raid_devices=False, + multi_attach=False, + ftt=None, ) -> str: + distr_ndcs, distr_npcs, max_fault_tolerance = _resolve_multi_attach( + multi_attach, ftt, raid_devices, ha_type, is_single_node, + distr_ndcs, distr_npcs, max_fault_tolerance) if (distr_ndcs, distr_npcs) not in SUPPORTED_ERASURE_CODING_SCHEMES: raise ValueError("Unsupported erasure coding scheme") _validate_integrity_flags(inline_checksum, atomic_4k) @@ -415,7 +457,9 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, if max_fault_tolerance > 1: if ha_type != "ha": raise ValueError("max_fault_tolerance > 1 requires ha_type='ha'") - if distr_npcs < 2: + # Multi-attach runs 1+0: its FTT comes from the array LUN being + # reopened on another node, not from parity. + if distr_npcs < 2 and not multi_attach: raise ValueError("max_fault_tolerance > 1 requires distr_npcs >= 2") if (hashicorp_vault_settings is not None) and (Settings().tls_connect != "authenticated"): @@ -570,6 +614,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.page_copy = bool(page_copy) cluster.md_journal = bool(md_journal) cluster.raid_devices = bool(raid_devices) + cluster.multi_attach = bool(multi_attach) cluster.client_data_nic = client_data_nic or "" cluster.max_fault_tolerance = max_fault_tolerance cluster.nvmf_base_port = nvmf_base_port @@ -688,6 +733,8 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn page_copy=False, md_journal=True, raid_devices=False, + multi_attach=False, + ftt=None, snode_api_service=constants.K8S_SNODE_API_SERVICE, spdk_proxy_service=constants.K8S_SPDK_PROXY_SERVICE, spdk_proxy_tls_secret=constants.K8S_SPDK_PROXY_TLS_SECRET, @@ -726,6 +773,8 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn page_copy=page_copy, md_journal=md_journal, raid_devices=raid_devices, + multi_attach=multi_attach, + ftt=ftt, snode_api_service=snode_api_service, spdk_proxy_service=spdk_proxy_service, spdk_proxy_tls_secret=spdk_proxy_tls_secret, @@ -764,6 +813,8 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca page_copy=False, md_journal=True, raid_devices=False, + multi_attach=False, + ftt=None, snode_api_service=constants.K8S_SNODE_API_SERVICE, spdk_proxy_service=constants.K8S_SPDK_PROXY_SERVICE, spdk_proxy_tls_secret=constants.K8S_SPDK_PROXY_TLS_SECRET, @@ -777,6 +828,9 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca if existing.cluster_name and existing.cluster_name == name: raise ValueError(f"A cluster with the name '{name}' already exists") + distr_ndcs, distr_npcs, max_fault_tolerance = _resolve_multi_attach( + multi_attach, ftt, raid_devices, ha_type, is_single_node, + distr_ndcs, distr_npcs, max_fault_tolerance) if (distr_ndcs, distr_npcs) not in SUPPORTED_ERASURE_CODING_SCHEMES: raise ValueError("Unsupported erasure coding scheme") @@ -786,7 +840,9 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca if max_fault_tolerance > 1: if ha_type != "ha": raise ValueError("max_fault_tolerance > 1 requires ha_type='ha'") - if distr_npcs < 2: + # Multi-attach runs 1+0: its FTT comes from the array LUN being + # reopened on another node, not from parity. + if distr_npcs < 2 and not multi_attach: raise ValueError("max_fault_tolerance > 1 requires distr_npcs >= 2") if (hashicorp_vault_settings is not None) and (Settings().tls_connect != "authenticated"): @@ -909,6 +965,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca cluster.page_copy = bool(page_copy) cluster.md_journal = bool(md_journal) cluster.raid_devices = bool(raid_devices) + cluster.multi_attach = bool(multi_attach) cluster.client_data_nic = client_data_nic or "" cluster.max_fault_tolerance = max_fault_tolerance cluster.nvmf_base_port = nvmf_base_port diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index f102c3909e..37a3771b8d 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -1045,6 +1045,29 @@ def get_config_var(name, default=None): # Flip to True once the ultra feat/affinity-floor data plane is deployed; # older data planes reject unknown RPC parameters. DISTR_AFFINITY_FLOOR_RPC = False + +# Multi-attach array LUN mode (cluster.multi_attach). None of these apply to +# a regular cluster. +# Peers' controllers to a multi-attach device hold I/O this long while no +# path is available (per-controller ctrlr_loss_timeout_sec), so a takeover +# that adds the LUN's new export as a path within the window resumes I/O +# instead of failing it. The cluster-wide bdev_nvme defaults (1 s) stay. +MULTI_ATTACH_HOLD_SEC = 120 +MULTI_ATTACH_RECONNECT_DELAY_SEC = 2 +MULTI_ATTACH_FAST_IO_FAIL_SEC = MULTI_ATTACH_HOLD_SEC +# The alceml of a multi-attach device fails its pending I/O and unregisters +# after this long without completion (bdev_alceml_create stuck_io_timeout_ms; +# the data-plane default is 4000 ms). It must exceed the multipath path +# failover below the LUN and stay below MULTI_ATTACH_HOLD_SEC, so a stuck +# owner instance is gone before the peers stop waiting. +MULTI_ATTACH_ALCEML_STUCK_IO_MS = 60000 +# Controller-ID ranges of multi-attach device subsystems: a node with +# cntlid_block i exports with cntlids [i*64+1, i*64+64]. SPDK multipath needs +# distinct cntlids across the paths of one controller, and a LUN may be +# exported by any node. Valid NVMe cntlids end at 0xFFEF. +MULTI_ATTACH_CNTLID_BLOCK_SIZE = 64 +MULTI_ATTACH_MAX_CNTLID = 0xFFEF +MULTI_ATTACH_MAX_NODES = 1022 MIG_JOB_SIZE = 64 # Sync-replication resync (tasks_runner_sync_resync): the shared task driver diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index f52aa965f1..d5db4be013 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -7,6 +7,7 @@ constants, distr_controller, jm_raid, + multi_attach, storage_node_ops, utils, ) @@ -665,6 +666,7 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False): checksum_method=checksum_method, cache_size=cache_size, cache_eviction_threshold=cache_eviction_threshold, + **multi_attach.alceml_create_kwargs(device_obj), ) except RPCException as e: logger.error("Failed to create alceml bdev %s: %s", alceml_name, e) @@ -685,7 +687,8 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False): else: logger.info(f"bdev already exists {pt_name}") - subsystem_nqn = snode.subsystem + ":dev:" + alceml_id + # Multi-attach: host-independent NQN (multi_attach.device_subsystem_nqn). + subsystem_nqn = multi_attach.device_subsystem_nqn(snode, device_obj, cluster) namespace_found = False subsys_found = False ret = rpc_client.subsystem_get(subsystem_nqn) @@ -699,7 +702,9 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False): if not subsys_found: logger.info("Creating subsystem %s", subsystem_nqn) - ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id) + ret = rpc_client.subsystem_create( + subsystem_nqn, 'sbcli-cn', alceml_id, + **multi_attach.subsystem_create_kwargs(snode, device_obj, db_controller)) for iface in snode.data_nics: if iface.ip4_address: ret = rpc_client.listeners_create(subsystem_nqn, iface.trtype, iface.listen_address, snode.nvmf_port) @@ -710,7 +715,13 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False): if not namespace_found: logger.info(f"Adding {pt_name} to the subsystem") - ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name) + if multi_attach.is_multi_attach(device_obj): + # Every node that exports the LUN must use the same namespace + # UUID: SPDK multipath joins paths by namespace identity. + ret = rpc_client.nvmf_subsystem_add_ns( + subsystem_nqn, pt_name, multi_attach.namespace_uuid(device_obj)) + else: + ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name) else: logger.info(f"bdev already added to subsys {pt_name}") diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index 8e65bcf402..833b849ada 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -1759,6 +1759,64 @@ def release_lvol_ns_slot(self, lvol): return transactional(self, kv, lvol) return lvol.remove(kv) + # ---- Multi-attach controller-ID block claim (Single FDB Transaction) ---- + + CNTLID_BLOCK_ALLOC_PREFIX = "cntlid_block_alloc" + + def _claim_cntlid_block_tx(self, tr, node_key, alloc_key, node_id, live_node_ids, + max_blocks, index_list): + # The per-cluster allocation map {node_id: block} is the authority: + # every claim reads and rewrites it, so two concurrent claims in one + # cluster conflict and the loser retries against the winner's entry. + raw = tr.get(alloc_key).wait() + alloc = json.loads(bytes(raw).decode()) if raw.present() else {} + if node_id in alloc: + block = int(alloc[node_id]) + else: + # Entries of nodes whose record is gone are reclaimed; a REMOVED + # node keeps its block (its record still exists). + alloc = {nid: b for nid, b in alloc.items() if nid in live_node_ids} + used = {int(b) for b in alloc.values()} + block = next((i for i in range(max_blocks) if i not in used), None) + if block is None: + raise ValueError( + f"no free controller-ID block: a multi-attach cluster supports " + f"at most {max_blocks} storage nodes") + alloc[node_id] = block + tr.set(alloc_key, json.dumps(alloc, sort_keys=True).encode()) + + def _set(node): + if node.cntlid_block == block: + return False + node.cntlid_block = block + return True + + self._atomic_update_tx(tr, node_key, StorageNode, _set, index_list) + return block + + def claim_cntlid_block(self, node, max_blocks): + """Claim the smallest free controller-ID block of ``node``'s cluster + for ``node`` (idempotent: a node that holds one gets it back) and + persist it on the node record, in ONE FDB transaction. Returns the + block index. + + Raises: + ValueError: every block of the cluster is taken. + """ + kv = self.kv_store + alloc_key = f"{self.CNTLID_BLOCK_ALLOC_PREFIX}/{node.cluster_id}".encode() + node_key = node.get_db_id().encode() + live_node_ids = {n.get_id() for n in self.get_storage_nodes_by_cluster_id(node.cluster_id)} + live_node_ids.add(node.get_id()) + index_list = StorageNode.active_indexes(kv) + if kv is not None and hasattr(kv, 'create_transaction'): + transactional = fdb.transactional(DBController._claim_cntlid_block_tx) + return transactional(self, kv, node_key, alloc_key, node.get_id(), + live_node_ids, max_blocks, index_list) + return self._claim_cntlid_block_tx( + _NoTxnStore(kv), node_key, alloc_key, node.get_id(), live_node_ids, + max_blocks, index_list) + # ---- Generic atomic read-modify-write (Single FDB Transaction) ---- def _atomic_update_tx(self, tr, key, model_cls, mutate_fn, index_list): diff --git a/simplyblock_core/distr_controller.py b/simplyblock_core/distr_controller.py index 427bcf2ffe..5e588b96e8 100644 --- a/simplyblock_core/distr_controller.py +++ b/simplyblock_core/distr_controller.py @@ -3,7 +3,7 @@ import re import threading -from simplyblock_core import utils +from simplyblock_core import multi_attach, utils from simplyblock_core.db_controller import DBController from simplyblock_core.models.nvme_device import NVMeDevice, RemoteDevice from simplyblock_core.models.storage_node import StorageNode @@ -407,10 +407,15 @@ def get_distr_cluster_map(snodes, target_node, distr_name="", home_site=None): dev_w_gib = utils.convert_size(dev.weight_size(), 'GiB') or 1 name = None dev_status = dev.status - if snode.get_id() == target_node.get_id(): - name = dev.alceml_bdev + own = snode.get_id() == target_node.get_id() + if own: local_node_index = index + if own and not multi_attach.is_multi_attach(dev): + name = dev.alceml_bdev else: + # A peer's device, or an own multi-attach device: the owner's + # distribs reach it through its loopback controller too, so a + # takeover path covers them as well. dev2 = target_remote_by_id.get(dev.get_id()) if dev2 is not None: name = dev2.remote_bdev @@ -766,7 +771,7 @@ def send_cluster_map_add_device(device: NVMeDevice, target_node: StorageNode): if target_node.status == StorageNode.STATUS_ONLINE: rpc_client = target_node.rpc_client(timeout=3) - if target_node.get_id() == dnode.get_id(): + if target_node.get_id() == dnode.get_id() and not multi_attach.is_multi_attach(device): name = device.alceml_bdev else: name = f"remote_{device.alceml_bdev}n1" diff --git a/simplyblock_core/floors.py b/simplyblock_core/floors.py index 1917e86434..92c156e4b6 100644 --- a/simplyblock_core/floors.py +++ b/simplyblock_core/floors.py @@ -20,6 +20,12 @@ Topology overrides: a single-node cluster needs 1 node and n+k data devices; a two-node (dual / arbitration) cluster needs 2 nodes. +Multi-attach array LUN clusters (``cluster.multi_attach``, always 1+0 on +``raid_devices``) replace the node floor by FTT+1 (at least 3 for FTT 2, +``cluster.max_fault_tolerance`` = FTT): an owner's LUN is reopened on one of +the surviving nodes. One data LUN per node is the recommended layout; only +>= 1 is enforced. The topology overrides do not apply to them. + Journal devices and the device model (floor 5) ---------------------------------------------- A node's journal lives in ``StorageNode.jm_device`` (a ``JMDevice``), never in @@ -71,6 +77,7 @@ class Floors: nodes_rule: str = "n+2k" data_devices_rule: str = "n+2k" failure_domains_rule: str = "n+k" + devices_per_node_rule: str = "data devices per node" def _flag(obj, name: str) -> bool: @@ -126,6 +133,20 @@ def cluster_floors(cluster, uses_failure_domains: bool, member_nodes: int | None data_devices, data_rule = n + 2 * k, "n+2k" nodes, nodes_rule = n + 2 * k, "n+2k" + if _flag(cluster, "multi_attach"): + ftt = _int(getattr(cluster, "max_fault_tolerance", 1)) + ftt = ftt if ftt in (1, 2) else 1 + return Floors( + nodes=max(ftt + 1, 3 if ftt == 2 else 0), + devices_per_node=1, + data_devices=n + k, + failure_domains=(n + k) if uses_failure_domains else None, + nodes_rule=f"FTT+1, multi-attach FTT {ftt}", + data_devices_rule="n+k, multi-attach", + devices_per_node_rule=("data devices per node; multi-attach: exactly one " + "data LUN per node is recommended"), + ) + if is_single_node_cluster(cluster, member_nodes): nodes, nodes_rule = 1, "single-node cluster" # No redundancy to rebuild onto a spare: the stripe alone. @@ -194,7 +215,7 @@ def floor_violations(floors: Floors, online_nodes: int, data_devices: int, if count < floors.devices_per_node: violations.append( f"node {node_id} has {count} online data devices < floor " - f"{floors.devices_per_node} (data devices per node)") + f"{floors.devices_per_node} ({floors.devices_per_node_rule})") if data_devices < floors.data_devices: violations.append( f"data devices online {data_devices} < floor {floors.data_devices} " diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index f62ff14283..835ceb360c 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -188,6 +188,13 @@ def is_topology_owned(self) -> bool: # no other device is writable (allow_device_colocation on the distribs, # affinity_floor "relaxed" on migrations). raid_devices: bool = False + # Cluster-create-time, immutable: multi-attach array LUN mode. Every data + # device is an array LUN presented to several storage nodes; the cluster + # runs 1+0 and tolerates max_fault_tolerance node failures by reopening + # an owner's LUN on another node and adding it as an extra NVMe-oF path. + # Requires raid_devices. Gates every multi-attach path (device NQN, + # controller-ID ranges, hold timeouts, owner loopback). + multi_attach: bool = False tls: bool = False tls_config: dict = default_factory(dict) is_re_balancing: bool = False diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index 76690d2031..adc4cbddbd 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -112,6 +112,26 @@ class NVMeDevice(BaseModel): # does not compute it; alerts_controller turns it into device_slow / # device_unhealthy while it is fresh. Empty until first reported. health_assessment: dict = default_factory(dict) + # Multi-attach array LUN mode (cluster.multi_attach). All "off" by + # default, so a device of a regular cluster is unchanged. + #: The device is an array LUN presented to several storage nodes; its + #: subsystem NQN is host-independent (``:lun:``) and + #: every distrib, the owner's included, reaches it as + #: ``remote_alceml_n1`` over NVMe-oF. + multi_attach: bool = False + #: World-wide name (or serial) of the LUN, the same on every node. + lun_wwn: str = "" + #: Node whose alceml currently serves the LUN; "" means the owner + #: (``node_id``). + serving_node_id: str = "" + #: node_id -> {"path": by-id path of the LUN on that node, "bdev": the + #: base bdev name there}. Only the owner's entry is filled so far. + node_paths: dict = default_factory(dict) + + def serving_node(self) -> str: + """The node whose alceml serves this device now (the owner unless a + multi-attach takeover moved it).""" + return self.serving_node_id or self.node_id def weight_size(self) -> int: """Bytes the cluster-map weight of this device is derived from: the diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 32d00b0634..e8a6f9cc95 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -230,6 +230,10 @@ class StorageNode(BaseNodeObject): # placement and emitted verbatim into the distrib cluster map for the data # plane. failure_domain: int = -1 + # Multi-attach array LUN clusters only: this node's stable controller-ID + # block (multi_attach.cntlid_range). -1 = not assigned; claimed at + # add-node or lazily when a multi-attach device subsystem is created. + cntlid_block: int = -1 # Sync-replication site of the node (operator-supplied at add-node). # Mandatory on a cluster with sync_replication, empty everywhere else. site: str = "" diff --git a/simplyblock_core/multi_attach.py b/simplyblock_core/multi_attach.py new file mode 100644 index 0000000000..0615d32c9c --- /dev/null +++ b/simplyblock_core/multi_attach.py @@ -0,0 +1,166 @@ +"""Multi-attach array LUN mode (``Cluster.multi_attach``), phase 2 helpers. + +On an array whose LUNs are already protected and presented to several storage +nodes, the cluster runs 1+0. Each node owns one data LUN; exactly one alceml +per LUN is active. Everything a peer (and, through a loopback attach, the owner +itself) uses to reach the LUN is an NVMe-oF controller ``remote_alceml_`` +whose subsystem NQN does not depend on the exporting host, so a later takeover +can add the LUN's export on another node as one more path of the same +controller: + +* the device subsystem NQN is ``:lun:`` + (SPDK refuses a path to a different subnqn); +* each node exports with its own controller-ID range (SPDK multipath refuses + duplicate cntlids across the paths of one controller); +* peers attach with a long per-controller hold, so I/O waits for a new path + instead of failing. + +Every helper here is a no-op / returns the regular value for a device or +cluster without the flag, so regular clusters are unchanged. +""" + +from __future__ import annotations + +import uuid as uuid_mod + +from simplyblock_core import constants + +#: The NQN infix that marks a host-independent multi-attach device subsystem. +LUN_NQN_INFIX = ":lun:" + + +def is_multi_attach(obj) -> bool: + """The multi-attach flag of a cluster or device, read defensively: a + record that predates the field or a test stand-in reads as False.""" + return getattr(obj, "multi_attach", False) is True + + +def device_nqn(cluster_nqn: str, device_id: str) -> str: + """The host-independent subsystem NQN of a multi-attach device: the base + of the cluster NQN (the part every node's ``snode.subsystem`` starts with) + plus ``:lun:``.""" + base = (cluster_nqn or constants.CLUSTER_NQN).split(":")[0] + return f"{base}{LUN_NQN_INFIX}{device_id}" + + +def device_subsystem_nqn(snode, device, cluster) -> str: + """The subsystem NQN a device is exported under: host-independent for a + multi-attach device, ``:dev:`` otherwise.""" + if is_multi_attach(device): + return device_nqn(getattr(cluster, "nqn", ""), device.get_id()) + return snode.subsystem + ":dev:" + device.get_id() + + +#: Namespace of the derived namespace UUIDs (``namespace_uuid``); fixed forever. +_LUN_NS_UUID_NAMESPACE = uuid_mod.UUID("6f1d3a52-8c4b-5e0a-9d27-3b8e4c1f7a90") + + +def namespace_uuid(device) -> str: + """The namespace UUID a device's PT bdev is exported with. + + Regular device: the device UUID, as before. Multi-attach device: a UUID + derived from the device UUID, the same on every node that exports the LUN + (SPDK multipath only joins paths whose namespaces are identical). It must + NOT be the device UUID itself: the alceml bdev is registered with the + device UUID, SPDK registers every bdev UUID as an alias, and the owner's + loopback controller (and a takeover node's controller to the LUN) would + then fail to register its namespace bdev with a duplicate alias. + """ + if not is_multi_attach(device): + return device.get_id() + return str(uuid_mod.uuid5(_LUN_NS_UUID_NAMESPACE, device.get_id())) + + +def cntlid_range(block: int) -> tuple[int, int]: + """``(min_cntlid, max_cntlid)`` of controller-ID block ``block``. + + Raises: + ValueError: ``block`` is negative or beyond the supported node count. + """ + if not isinstance(block, int) or block < 0 or block >= constants.MULTI_ATTACH_MAX_NODES: + raise ValueError( + f"controller-ID block {block!r} out of range [0, {constants.MULTI_ATTACH_MAX_NODES})") + size = constants.MULTI_ATTACH_CNTLID_BLOCK_SIZE + lo, hi = block * size + 1, block * size + size + if hi > constants.MULTI_ATTACH_MAX_CNTLID: + raise ValueError(f"controller-ID block {block} exceeds cntlid 0x{constants.MULTI_ATTACH_MAX_CNTLID:X}") + return lo, hi + + +def ensure_cntlid_block(snode, db_controller=None) -> int: + """The node's controller-ID block, claimed (smallest free index of the + cluster, persisted on the node record) when it has none yet. Sets + ``snode.cntlid_block`` in memory too, so a later full write of this + object keeps it.""" + block = getattr(snode, "cntlid_block", -1) + if isinstance(block, int) and block >= 0: + return block + if db_controller is None: + from simplyblock_core.db_controller import DBController + db_controller = DBController() + block = db_controller.claim_cntlid_block(snode, constants.MULTI_ATTACH_MAX_NODES) + snode.cntlid_block = block + return block + + +def subsystem_create_kwargs(snode, device, db_controller=None) -> dict: + """Extra ``rpc_client.subsystem_create`` kwargs for a device subsystem: + the node's controller-ID range for a multi-attach device, nothing + otherwise (the regular min_cntlid=1 default stays).""" + if not is_multi_attach(device): + return {} + lo, hi = cntlid_range(ensure_cntlid_block(snode, db_controller)) + return {"min_cntlid": lo, "max_cntlid": hi} + + +def attach_controller_kwargs(device) -> dict: + """Per-controller hold timeouts for a peer's controller to a multi-attach + device (``bdev_nvme_attach_controller``); nothing otherwise, so the + cluster-wide bdev_nvme options apply as before.""" + if not is_multi_attach(device): + return {} + return { + "ctrlr_loss_timeout_sec": constants.MULTI_ATTACH_HOLD_SEC, + "reconnect_delay_sec": constants.MULTI_ATTACH_RECONNECT_DELAY_SEC, + "fast_io_fail_timeout_sec": constants.MULTI_ATTACH_FAST_IO_FAIL_SEC, + } + + +def alceml_create_kwargs(device) -> dict: + """Extra ``bdev_alceml_create`` kwargs for a multi-attach device's alceml + (the stuck-I/O timeout); nothing otherwise.""" + if not is_multi_attach(device): + return {} + return {"stuck_io_timeout_ms": constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS} + + +def mark_lun_devices(cluster, snode, devices, blk_entries) -> None: + """Flag the lblk devices of a multi-attach cluster as array LUNs: set + ``multi_attach``, the LUN WWN (the inventory's WWN, else its serial, which + lsblk falls back to the WWN for) and the owner's ``node_paths`` entry. + No-op on any other cluster. An entry configured as the journal is left + unflagged; a device the add-node path later picks as the journal keeps + the flag but never uses it (the journal stack does not read it).""" + if not is_multi_attach(cluster): + return + by_serial = {e.get("serial"): e for e in (blk_entries or [])} + for dev in devices: + entry = by_serial.get(dev.serial_number, {}) + if entry.get("journal"): + continue + dev.multi_attach = True + dev.lun_wwn = entry.get("wwn") or dev.serial_number or "" + dev.node_paths = { + snode.get_id(): { + "path": dev.by_id_path or dev.device_path, + "bdev": dev.nvme_bdev, + } + } + + +def own_multi_attach_devices(node, statuses) -> list: + """The node's own multi-attach data devices in one of ``statuses``: the + devices its distribs reach through a loopback ``remote_alceml_n1`` + controller instead of the local alceml.""" + return [d for d in (getattr(node, "nvme_devices", None) or []) + if is_multi_attach(d) and d.status in statuses and d.alceml_bdev] diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index b882af492f..72afdfc6d0 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -477,7 +477,8 @@ def subsystem_get(self, nqn: str) -> dict | None: def subsystem_delete(self, nqn): return self._request("nvmf_delete_subsystem", nqn=nqn) - def subsystem_create(self, nqn, serial_number, model_number, min_cntlid=1, max_namespaces=32, allow_any_host=True): + def subsystem_create(self, nqn, serial_number, model_number, min_cntlid=1, max_namespaces=32, allow_any_host=True, + max_cntlid=None): params = { "nqn": nqn, "serial_number": serial_number, @@ -486,6 +487,10 @@ def subsystem_create(self, nqn, serial_number, model_number, min_cntlid=1, max_n "ana_reporting": True, "max_namespaces": max_namespaces, "model_number": model_number} + # Sent only when given (multi-attach device subsystems: a per-node + # controller-ID range), so every other call is unchanged. + if max_cntlid is not None: + params["max_cntlid"] = int(max_cntlid) return self._request("nvmf_create_subsystem", **params) def keyring_file_add_key(self, name: str, path: str, *, allow_existing: bool = False): @@ -967,7 +972,7 @@ def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3, alceml_cpu_mask="", alceml_worker_cpu_mask="", pba_page_size=2097152, write_protection=False, full_page_unmap=False, checksum_method=0, cache_size=0, cache_eviction_threshold=0, - force_4k_atomic=False): + force_4k_atomic=False, stuck_io_timeout_ms=0): params = { "name": alceml_name, "cntr_path": nvme_name, @@ -1004,6 +1009,11 @@ def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3, # atomicity (cluster.atomic_4k). Tell the data plane to skip its >=4K # block-size gate for fallback-mode checksum validation. params["cv_ignore_block_size"] = bool(force_4k_atomic) + # Fail pending I/O and unregister after this long without completion + # (data-plane default 4000 ms). Sent only when non-zero: multi-attach + # devices, whose array LUN needs longer than multipath failover. + if stuck_io_timeout_ms: + params["stuck_io_timeout_ms"] = int(stuck_io_timeout_ms) return self._request("bdev_alceml_create", **params) def bdev_distrib_create(self, name, vuid, ndcs, npcs, num_blocks, block_size, jm_names, diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 0914135746..0d767a6a54 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -38,6 +38,7 @@ distr_controller, floors, jm_raid, + multi_attach, scripts, storage_network, utils, @@ -1205,6 +1206,11 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r # ``"failover"`` (active-passive) and remote alceml/jm controllers # would carry all IO on a single path. attach_mode = "multipath" if device.nvmf_multipath else False + if multi_attach.is_multi_attach(device): + # A multi-attach device's controller must accept a further path (the + # LUN's export on a takeover node), which a controller attached with + # multipath disabled refuses. + attach_mode = "multipath" final = rpc_client.bdev_nvme_controller_list(name) if not final: @@ -1220,7 +1226,8 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r try: resp = attach_rpc_client.bdev_nvme_attach_controller( name, device.nvmf_nqn, ip, device.nvmf_port, tr_type, - multipath=attach_mode) + multipath=attach_mode, + **multi_attach.attach_controller_kwargs(device)) if resp: dial_backoff.record_success(ip) else: @@ -1275,7 +1282,8 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r try: if attach_rpc_client.bdev_nvme_attach_controller( name, device.nvmf_nqn, ip, device.nvmf_port, tr_type, - multipath=attach_mode): + multipath=attach_mode, + **multi_attach.attach_controller_kwargs(device)): dial_backoff.record_success(ip) else: dial_backoff.record_failure(ip) @@ -1411,7 +1419,8 @@ def _repair_multipath_controller_locked(node, name, device, rpc_client, ret, # the 2026-08-20 soak, all of them "-5 Input/output error". if not rpc_client.bdev_nvme_attach_controller( name, device.nvmf_nqn, ip, device.nvmf_port, - tr_type, multipath="multipath"): + tr_type, multipath="multipath", + **multi_attach.attach_controller_kwargs(device)): dial_backoff.record_failure(ip) logger.warning( "Re-attach of path %s on controller %s was rejected by the " @@ -2015,6 +2024,7 @@ def _create_storage_device_stack(rpc_client, nvme, snode: StorageNode, after_res cache_size=cache_size, cache_eviction_threshold=cache_eviction_threshold, force_4k_atomic=getattr(cluster, 'atomic_4k', False), + **multi_attach.alceml_create_kwargs(nvme), ) if not ret: @@ -2034,9 +2044,12 @@ def _create_storage_device_stack(rpc_client, nvme, snode: StorageNode, after_res logger.error(f"Failed to read back pt bdev: {pt_name}") return None pt_spdk_uuid = pt_bdev_info["aliases"][0] - subsystem_nqn = snode.subsystem + ":dev:" + alceml_id + # Multi-attach: host-independent NQN and this node's cntlid range, so the + # LUN's export on another node can join the same controller as a path. + subsystem_nqn = multi_attach.device_subsystem_nqn(snode, nvme, cluster) logger.info("creating subsystem %s", subsystem_nqn) - ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id) + ret = rpc_client.subsystem_create(subsystem_nqn, 'sbcli-cn', alceml_id, + **multi_attach.subsystem_create_kwargs(snode, nvme, db_controller)) ip_list = [] for iface in snode.data_nics: if iface.ip4_address: @@ -2045,7 +2058,7 @@ def _create_storage_device_stack(rpc_client, nvme, snode: StorageNode, after_res ip_list.append(iface.ip4_address) logger.info(f"add {pt_name} to subsystem") - ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name, alceml_id) + ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name, multi_attach.namespace_uuid(nvme)) if not ret: logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}") return None @@ -2500,14 +2513,22 @@ def _connect_to_remote_devs( devices_to_connect = [] connect_threads = [] nodes = db_controller.get_storage_nodes_by_cluster_id(this_node.cluster_id) + # Multi-attach clusters: the node's own array-LUN devices are reached by + # its distribs through a loopback controller (remote_alceml_n1) like + # every peer's, so they are connected here too. Empty on other clusters. + own_loopback = multi_attach.own_multi_attach_devices(this_node, allowed_dev_statuses) # connect to remote devs for node_index, node in enumerate(nodes): - if node.get_id() == this_node.get_id() or node.status not in allowed_node_statuses: + if node.get_id() == this_node.get_id(): + if not own_loopback: + continue + elif node.status not in allowed_node_statuses: continue if only_node_id and node.get_id() != only_node_id: continue logger.info(f"Connecting to node {node.get_id()}") - for index, dev in enumerate(node.nvme_devices): + node_devices = own_loopback if node.get_id() == this_node.get_id() else node.nvme_devices + for index, dev in enumerate(node_devices): if dev.status not in allowed_dev_statuses: logger.debug(f"Device is not online: {dev.get_id()}, status: {dev.status}") @@ -2613,7 +2634,10 @@ def _find_remote_bdev(dev): # to per-device probes. _sweep_bdev_names = _fetch_bdev_name_set(rpc_client) for node in nodes: - if node.get_id() == this_node.get_id() or node.status not in allowed_node_statuses: + if node.get_id() == this_node.get_id(): + if not own_loopback: + continue + elif node.status not in allowed_node_statuses: continue if only_node_id and node.get_id() != only_node_id: # DELTA mode: records for every other node were carried over @@ -2623,7 +2647,7 @@ def _find_remote_bdev(dev): # ~25 min, 2026-07-13 validation run). Verify only the delta # owner's devices. continue - for dev in node.nvme_devices: + for dev in (own_loopback if node.get_id() == this_node.get_id() else node.nvme_devices): if dev.get_id() in remote_device_ids: continue if dev.status not in allowed_dev_statuses: @@ -2764,6 +2788,12 @@ def _verify_online_device_coverage(snode: StorageNode, repair: bool = True): if not dev.alceml_bdev: continue expected[f"remote_{dev.alceml_bdev}n1"] = dev + # Multi-attach clusters: the node's own array LUNs are mapped through its + # loopback controller, so they need one too (empty on other clusters). + for dev in multi_attach.own_multi_attach_devices( + snode, (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE)): + expected[f"remote_{dev.alceml_bdev}n1"] = dev if fd_recovery and excluded_fd_devs: # Never a silent relaxation — degraded recovery must be visible in logs. @@ -2836,10 +2866,17 @@ def sync_remote_devices_from_spdk(this_node: StorageNode): for peer in db_controller.get_storage_nodes_by_cluster_id(fresh_node.cluster_id): if peer.get_id() == fresh_node.get_id(): + # Multi-attach clusters: own array LUNs (loopback controllers). + peer_devices = multi_attach.own_multi_attach_devices( + fresh_node, (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE)) + if not peer_devices: + continue + elif peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN, StorageNode.STATUS_RESTARTING]: continue - if peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN, StorageNode.STATUS_RESTARTING]: - continue - for dev in peer.nvme_devices: + else: + peer_devices = peer.nvme_devices + for dev in peer_devices: if dev.status not in [ NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, @@ -2916,11 +2953,18 @@ def reconnect_dropped_remote_devs(this_node: StorageNode): for peer in db_controller.get_storage_nodes_by_cluster_id(fresh_node.cluster_id): if peer.get_id() == fresh_node.get_id(): + # Multi-attach clusters: own array LUNs (loopback controllers). + peer_devices = multi_attach.own_multi_attach_devices( + fresh_node, (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE)) + if not peer_devices: + continue + elif peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN, + StorageNode.STATUS_UNREACHABLE]: continue - if peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN, - StorageNode.STATUS_UNREACHABLE]: - continue - for dev in peer.nvme_devices: + else: + peer_devices = peer.nvme_devices + for dev in peer_devices: if dev.get_id() in known_ids: continue if dev.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, @@ -4866,6 +4910,10 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, return False snode.write_to_db(kv_store) + if multi_attach.is_multi_attach(cluster): + # Stable controller-ID block of this node for its multi-attach + # device subsystems (smallest free index of the cluster). + multi_attach.ensure_cntlid_block(snode, db_controller) try: rpc_client.nvmf_set_max_subsystems(constants.NVMF_MAX_SUBSYSTEMS) @@ -4979,6 +5027,8 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, # discover devices if lblk_mode: nvme_devs = utils.addAioDevices(rpc_client, snode, lblk_resolved) + # Multi-attach clusters only: flag the array LUNs (no-op otherwise). + multi_attach.mark_lun_devices(cluster, snode, nvme_devs, lblk_resolved) else: if not snode.ssd_pcie: node_info, _ = snode_api.info() @@ -9708,6 +9758,9 @@ def _restart_storage_node_impl( if not nvme_devs: logger.error("No eligible block devices were found!") return False + # Multi-attach clusters only: a device first seen on this restart + # (new host) is an array LUN too. Existing records keep their flags. + multi_attach.mark_lun_devices(cluster, snode, nvme_devs, lblk_resolved) else: # After a host reboot the node's NVMe devices are back on the kernel # nvme driver, so spdk_pcie_list (devices bound to uio/vfio) is empty diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 9a819725cd..d0fa05313d 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1633,6 +1633,9 @@ def resolve_lblk_entries(configured_entries, host_devices): } if path: resolved_entry["path"] = path + if live.get("wwn"): + # LUN identity of a multi-attach (array LUN) device. + resolved_entry["wwn"] = live["wwn"] if entry.get("type") == "part" or live.get("type") == "part": resolved_entry["type"] = "part" resolved_entry["partuuid"] = live.get("partuuid") or entry.get("partuuid", "") diff --git a/simplyblock_web/api/v1/cluster.py b/simplyblock_web/api/v1/cluster.py index dfa707d1f3..c21f0a2a91 100644 --- a/simplyblock_web/api/v1/cluster.py +++ b/simplyblock_web/api/v1/cluster.py @@ -52,6 +52,8 @@ def add_cluster(): inflight_io_threshold = cl_data.get('inflight_io_threshold', 4) strict_node_anti_affinity = cl_data.get('strict_node_anti_affinity', False) raid_devices = bool(cl_data.get('raid_devices', False)) + multi_attach = bool(cl_data.get('multi_attach', False)) + ftt = cl_data.get('ftt') is_single_node = cl_data.get('is_single_node', False) client_data_nic = cl_data.get('client_data_nic', "") max_fault_tolerance = min(distr_npcs, 2) if distr_npcs >= 1 else 1 @@ -68,6 +70,8 @@ def add_cluster(): max_fault_tolerance=max_fault_tolerance, backup_config=backup_config, nvmf_base_port=nvmf_base_port, rpc_base_port=rpc_base_port, snode_api_port=snode_api_port, raid_devices=raid_devices, + multi_attach=multi_attach, + ftt=ftt, hashicorp_vault_settings=( HashicorpVaultSettings(raw_vault) if (raw_vault := cl_data.get('hashicorp_vault_settings')) is not None diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index b72e16d0e1..804c971c31 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -211,6 +211,7 @@ class ClusterDTO(BaseModel): page_copy: bool = False md_journal: bool = False raid_devices: bool = False + multi_attach: bool = False secret: SecretStr tls_enabled: bool max_fault_tolerance: int @@ -263,6 +264,7 @@ def from_model(model: Cluster, stat_obj: StatsObject | None = None): page_copy=model.page_copy, md_journal=model.md_journal, raid_devices=model.raid_devices, + multi_attach=model.multi_attach, secret=model.secret, tls_enabled=model.tls, max_fault_tolerance=model.max_fault_tolerance, diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index 1bb9845af6..f223b2666f 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -88,6 +88,11 @@ class ClusterParams(BaseModel): # Every data device is protected below simplyblock (hardware RAID LUN, md # or dm RAID): data-device floor n+k instead of n+2k. Immutable. raid_devices: bool = False + # Multi-attach array LUN mode (1+0, FTT from LUN takeover). Requires + # raid_devices, ha_type 'ha' and ftt. Immutable. + multi_attach: bool = False + #: Node failures to tolerate in multi-attach mode; only with multi_attach. + ftt: Literal[1, 2] | None = None is_single_node: bool = False fabric: str = "tcp" cr_name: str = "" diff --git a/tests/unit/test_multi_attach.py b/tests/unit/test_multi_attach.py new file mode 100644 index 0000000000..697455463a --- /dev/null +++ b/tests/unit/test_multi_attach.py @@ -0,0 +1,621 @@ +"""Multi-attach array LUN mode, phase 2 (simplyblock_core.multi_attach). + + - cluster create/add options: --multi-attach needs --raid-devices, --ftt, + ha_type 'ha'; forces 1+0 and max_fault_tolerance = ftt; --ftt alone is + refused; the regular npcs>=2 rule for FTT2 does not block multi-attach. + - floors: nodes >= FTT+1 (>= 3 for FTT2), >= 1 data device per node. + - device model defaults and the lblk LUN marking. + - host-independent NQN, derived namespace UUID, per-node cntlid ranges. + - RPC wrappers: max_cntlid and stuck_io_timeout_ms sent only when given. + - device stacks (storage_node_ops and device_controller): NQN, cntlid + range, namespace UUID, alceml stuck-I/O timeout; unchanged otherwise. + - peer attach: hold timeouts and multipath mode only for multi-attach. + - owner loopback: cluster map and remote-device connect. +""" + +import json +import unittest +import uuid as uuid_mod +from unittest.mock import MagicMock, patch + +from simplyblock_core import ( + cluster_ops, + constants, + distr_controller, + floors, + multi_attach, + storage_node_ops, +) +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.rpc_client import RPCClient + +DEV_ID = "1b4e28ba-2fa1-41d2-883f-0016d3cca427" + + +def _device(dev_id=DEV_ID, multi=False, node_id="node-a", order=0): + d = NVMeDevice() + d.uuid = dev_id + d.node_id = node_id + d.status = NVMeDevice.STATUS_ONLINE + d.alceml_bdev = f"alceml_{dev_id}" + d.alceml_name = f"alceml_{dev_id}" + d.cluster_device_order = order + d.size = 8 * 1024 ** 3 + d.multi_attach = multi + return d + + +def _ma_cluster(ftt=1): + cl = Cluster() + cl.uuid = "cluster-1" + cl.nqn = f"{constants.CLUSTER_NQN}:cluster-1" + cl.distr_ndcs, cl.distr_npcs = 1, 0 + cl.raid_devices = True + cl.multi_attach = True + cl.ha_type = "ha" + cl.max_fault_tolerance = ftt + cl.status = Cluster.STATUS_ACTIVE + return cl + + +# ---------------------------------------------------------------- options + +class TestResolveOptions(unittest.TestCase): + + def _resolve(self, **kw): + args = dict(multi_attach=True, ftt=1, raid_devices=True, ha_type="ha", + is_single_node=False, distr_ndcs=2, distr_npcs=1, + max_fault_tolerance=1) + args.update(kw) + return cluster_ops._resolve_multi_attach(**args) + + def test_regular_cluster_is_unchanged(self): + self.assertEqual(self._resolve(multi_attach=False, ftt=None), (2, 1, 1)) + + def test_ftt_without_multi_attach_is_refused(self): + with self.assertRaisesRegex(ValueError, "only valid together with multi_attach"): + self._resolve(multi_attach=False, ftt=2) + + def test_forces_one_plus_zero_and_ftt(self): + self.assertEqual(self._resolve(ftt=1), (1, 0, 1)) + self.assertEqual(self._resolve(ftt=2, distr_ndcs=4, distr_npcs=2), (1, 0, 2)) + + def test_requires_raid_devices(self): + with self.assertRaisesRegex(ValueError, "raid_devices"): + self._resolve(raid_devices=False) + + def test_requires_ftt(self): + for ftt in (None, 0, 3): + with self.subTest(ftt=ftt), self.assertRaisesRegex(ValueError, "ftt 1 or 2"): + self._resolve(ftt=ftt) + + def test_requires_ha(self): + with self.assertRaisesRegex(ValueError, "ha_type='ha'"): + self._resolve(ha_type="single") + + def test_refuses_single_node(self): + with self.assertRaisesRegex(ValueError, "single-node"): + self._resolve(is_single_node=True) + + +def _add_cluster(**kw): + """Run _add_cluster_impl with the persistence and side effects stubbed; + returns the Cluster it wrote.""" + written = [] + db = MagicMock() + db.get_clusters.return_value = [MagicMock(cluster_name="other")] + cfg = MagicMock() + cfg.mode = "docker" + cfg.db_connection = "" + cfg.grafana_secret = cluster_ops.SecretStr("") + cfg.grafana_endpoint = "" + cfg.disable_monitoring = True + db.get_deploy_config.return_value = cfg + args = dict(blk_size=4096, page_size_in_blocks=2097152, cap_warn=0, cap_crit=0, + prov_cap_warn=0, prov_cap_crit=0, distr_ndcs=1, distr_npcs=1, + distr_bs=4096, distr_chunk_bs=4096, ha_type="ha", enable_node_affinity=False, + qpair_count=256, max_queue_size=128, inflight_io_threshold=4, + strict_node_anti_affinity=False, is_single_node=False, name="", + max_subsys=10, spdk_vcpu_count=4) + args.update(kw) + with patch.object(cluster_ops, "db_controller", db), \ + patch.object(cluster_ops, "cluster_events"), \ + patch.object(cluster_ops, "validate_spdk_sizing"), \ + patch.object(Cluster, "write_to_db", autospec=True, + side_effect=lambda self, *a, **k: written.append(self)): + cluster_ops._add_cluster_impl(**args) + return written[-1] + + +class TestAddCluster(unittest.TestCase): + + def test_multi_attach_ftt2_is_stored_as_one_plus_zero(self): + cl = _add_cluster(multi_attach=True, ftt=2, raid_devices=True, + distr_ndcs=1, distr_npcs=1, max_fault_tolerance=1) + self.assertTrue(cl.multi_attach) + self.assertTrue(cl.raid_devices) + self.assertEqual((cl.distr_ndcs, cl.distr_npcs), (1, 0)) + self.assertEqual(cl.max_fault_tolerance, 2) + + def test_regular_cluster_keeps_its_scheme(self): + cl = _add_cluster(distr_ndcs=2, distr_npcs=1, max_fault_tolerance=1) + self.assertFalse(cl.multi_attach) + self.assertEqual((cl.distr_ndcs, cl.distr_npcs, cl.max_fault_tolerance), (2, 1, 1)) + + def test_regular_ftt2_still_needs_two_parity_chunks(self): + with self.assertRaisesRegex(ValueError, "distr_npcs >= 2"): + _add_cluster(distr_ndcs=2, distr_npcs=1, max_fault_tolerance=2) + + def test_multi_attach_without_raid_devices_is_refused(self): + with self.assertRaisesRegex(ValueError, "raid_devices"): + _add_cluster(multi_attach=True, ftt=1) + + def test_add_cluster_passes_the_options_through(self): + with patch.object(cluster_ops, "_add_cluster_impl", return_value="c1") as impl: + cluster_ops.add_cluster(4096, 2097152, 0, 0, 0, 0, 1, 1, 4096, 4096, "ha", False, + 256, 128, 4, False, False, "", multi_attach=True, ftt=2, + raid_devices=True) + kw = impl.call_args.kwargs + self.assertEqual((kw["multi_attach"], kw["ftt"]), (True, 2)) + + def test_create_cluster_refuses_ftt_alone(self): + with self.assertRaisesRegex(ValueError, "only valid together"): + cluster_ops.create_cluster( + 4096, 2097152, "", 0, 0, 0, 0, "eth0", "10.0.0.1", 0, 0, "", "", 1, 1, + 4096, 4096, "ha", "docker", False, 256, 256, 128, 4, True, False, "", + "", "", "", "tcp", False, "", ftt=1) + + +class TestCli(unittest.TestCase): + + def _parse(self, *argv): + from simplyblock_cli import cli as cli_module + wrapper = cli_module.CLIWrapper.__new__(cli_module.CLIWrapper) + cli_module.CLIWrapper.__init__(wrapper) + return wrapper.parser.parse_args(list(argv)) + + def test_options_on_create_and_add(self): + for command in ("create", "add"): + with self.subTest(command=command): + args = self._parse("cluster", command, "--raid-devices", "--multi-attach", "--ftt", "2") + self.assertTrue(args.multi_attach) + self.assertEqual(args.ftt, 2) + plain = self._parse("cluster", command) + self.assertFalse(plain.multi_attach) + self.assertIsNone(plain.ftt) + + def test_ftt_choices(self): + with self.assertRaises(SystemExit): + self._parse("cluster", "add", "--ftt", "3") + + +class TestApiV2(unittest.TestCase): + + def test_params_carry_the_options(self): + from simplyblock_web.api.v2.cluster import ClusterParams + p = ClusterParams(max_subsys=10, spdk_vcpu_count=4, raid_devices=True, + multi_attach=True, ftt=2) + dumped = p.model_dump(exclude_none=True) + self.assertEqual((dumped["multi_attach"], dumped["ftt"]), (True, 2)) + self.assertNotIn("ftt", ClusterParams(max_subsys=10, spdk_vcpu_count=4).model_dump(exclude_none=True)) + + +# ---------------------------------------------------------------- floors + +def _node(uid, devices=1, status=StorageNode.STATUS_ONLINE): + n = StorageNode() + n.uuid = uid + n.cluster_id = "cluster-1" + n.status = status + n.nvme_devices = [_device(f"{uid}-d{i}", multi=True, node_id=uid) for i in range(devices)] + return n + + +class TestFloors(unittest.TestCase): + + def test_node_floor_is_ftt_plus_one(self): + f1 = floors.cluster_floors(_ma_cluster(1), False, member_nodes=2) + self.assertEqual((f1.nodes, f1.devices_per_node, f1.data_devices), (2, 1, 1)) + f2 = floors.cluster_floors(_ma_cluster(2), False, member_nodes=2) + self.assertEqual(f2.nodes, 3) + self.assertIn("multi-attach", f2.nodes_rule) + + def test_no_single_or_two_node_override(self): + self.assertEqual(floors.cluster_floors(_ma_cluster(2), False, member_nodes=1).nodes, 3) + + def test_violations_name_the_rule(self): + cl = _ma_cluster(2) + v = floors.evaluate(cl, [_node("a"), _node("b")]) + self.assertTrue(any("FTT+1" in x for x in v), v) + v = floors.evaluate(cl, [_node("a"), _node("b"), _node("c", devices=0)]) + self.assertEqual(len(v), 1) + self.assertIn("one data LUN per node is recommended", v[0]) + self.assertEqual(floors.evaluate(cl, [_node("a"), _node("b", devices=2), _node("c")]), []) + + def test_regular_violation_text_unchanged(self): + cl = _ma_cluster(1) + cl.multi_attach = False + cl.distr_ndcs, cl.distr_npcs = 1, 1 + v = floors.evaluate(cl, [_node("a"), _node("b"), _node("c"), _node("d", devices=0)]) + self.assertIn("node d has 0 online data devices < floor 1 (data devices per node)", v) + + +# ------------------------------------------------------- device model/NQN + +class TestDeviceModel(unittest.TestCase): + + def test_defaults_are_off(self): + d = NVMeDevice() + self.assertEqual((d.multi_attach, d.lun_wwn, d.serving_node_id, d.node_paths), + (False, "", "", {})) + d.node_id = "n1" + self.assertEqual(d.serving_node(), "n1") + d.serving_node_id = "n2" + self.assertEqual(d.serving_node(), "n2") + + def test_record_round_trips(self): + d = _device(multi=True) + d.lun_wwn = "0x600a" + d.node_paths = {"node-a": {"path": "/dev/disk/by-id/wwn-0x600a", "bdev": "aio_x"}} + back = NVMeDevice(json.loads(json.dumps(d.to_dict()))) + self.assertEqual((back.multi_attach, back.lun_wwn, back.node_paths), + (True, "0x600a", d.node_paths)) + + def _devs(self): + a = NVMeDevice({"uuid": "a", "serial_number": "SER-A", "nvme_bdev": "aio_a", + "by_id_path": "/dev/disk/by-id/wwn-0x1", "device_path": "/dev/sdb"}) + b = NVMeDevice({"uuid": "b", "serial_number": "SER-B", "nvme_bdev": "aio_b", + "device_path": "/dev/sdc"}) + j = NVMeDevice({"uuid": "j", "serial_number": "SER-J", "nvme_bdev": "aio_j"}) + entries = [{"serial": "SER-A", "wwn": "0x1"}, {"serial": "SER-B"}, + {"serial": "SER-J", "journal": True}] + return a, b, j, entries + + def test_mark_lun_devices(self): + a, b, j, entries = self._devs() + snode = MagicMock() + snode.get_id.return_value = "node-a" + multi_attach.mark_lun_devices(_ma_cluster(), snode, [a, b, j], entries) + self.assertTrue(a.multi_attach and b.multi_attach) + self.assertEqual((a.lun_wwn, b.lun_wwn), ("0x1", "SER-B")) + self.assertEqual(a.node_paths, {"node-a": {"path": "/dev/disk/by-id/wwn-0x1", "bdev": "aio_a"}}) + self.assertEqual(b.node_paths["node-a"]["path"], "/dev/sdc") + self.assertFalse(j.multi_attach) + + def test_mark_is_a_no_op_on_a_regular_cluster(self): + a, b, j, entries = self._devs() + cl = _ma_cluster() + cl.multi_attach = False + multi_attach.mark_lun_devices(cl, MagicMock(), [a, b], entries) + self.assertFalse(a.multi_attach or b.multi_attach) + self.assertEqual((a.lun_wwn, a.node_paths), ("", {})) + + def test_resolve_lblk_entries_carries_the_wwn(self): + from simplyblock_core import utils + live = {"name": "sdb", "device_path": "/dev/sdb", "serial": "SER-A", "wwn": "0x1", + "by_id_path": "/dev/disk/by-id/wwn-0x1", "size": 1, "type": "disk"} + resolved, missing = utils.resolve_lblk_entries([{"serial": "SER-A", "name": "sdb"}], [live]) + self.assertEqual((resolved[0]["wwn"], missing), ("0x1", [])) + + +class TestNames(unittest.TestCase): + + def test_device_nqn_is_host_independent(self): + cl = _ma_cluster() + snode = MagicMock(subsystem=f"{constants.CLUSTER_NQN}:host1_8080") + nqn = multi_attach.device_subsystem_nqn(snode, _device(multi=True), cl) + self.assertEqual(nqn, f"{constants.CLUSTER_NQN}:lun:{DEV_ID}") + self.assertEqual(multi_attach.device_subsystem_nqn(snode, _device(), cl), + f"{constants.CLUSTER_NQN}:host1_8080:dev:{DEV_ID}") + + def test_namespace_uuid(self): + self.assertEqual(multi_attach.namespace_uuid(_device()), DEV_ID) + ns = multi_attach.namespace_uuid(_device(multi=True)) + self.assertNotEqual(ns, DEV_ID) + self.assertEqual(ns, multi_attach.namespace_uuid(_device(multi=True))) + uuid_mod.UUID(ns) + + def test_cntlid_ranges(self): + self.assertEqual(multi_attach.cntlid_range(0), (1, 64)) + self.assertEqual(multi_attach.cntlid_range(1), (65, 128)) + lo, hi = multi_attach.cntlid_range(constants.MULTI_ATTACH_MAX_NODES - 1) + self.assertLessEqual(hi, 0xFFEF) + for bad in (-1, constants.MULTI_ATTACH_MAX_NODES): + with self.subTest(block=bad), self.assertRaises(ValueError): + multi_attach.cntlid_range(bad) + + def test_kwargs_are_empty_for_regular_devices(self): + d = _device() + self.assertEqual(multi_attach.attach_controller_kwargs(d), {}) + self.assertEqual(multi_attach.alceml_create_kwargs(d), {}) + self.assertEqual(multi_attach.subsystem_create_kwargs(MagicMock(), d), {}) + self.assertEqual(multi_attach.attach_controller_kwargs(MagicMock()), {}) + + def test_hold_and_stuck_io_constants(self): + self.assertLess(constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS, constants.MULTI_ATTACH_HOLD_SEC * 1000) + self.assertEqual(multi_attach.attach_controller_kwargs(_device(multi=True)), { + "ctrlr_loss_timeout_sec": 120, "reconnect_delay_sec": 2, "fast_io_fail_timeout_sec": 120}) + self.assertEqual(multi_attach.alceml_create_kwargs(_device(multi=True)), + {"stuck_io_timeout_ms": 60000}) + + +class TestCntlidBlockClaim(unittest.TestCase): + + def _claim(self, alloc, node_id="n3", live=("n1", "n2", "n3")): + from simplyblock_core.db_controller import DBController, _NoTxnStore + kv = {} + if alloc is not None: + kv[b"alloc"] = json.dumps(alloc).encode() + store = MagicMock() + store.get.side_effect = kv.get + store.set.side_effect = kv.__setitem__ + db = DBController.__new__(DBController) + node = StorageNode() + node.uuid = node_id + with patch.object(DBController, "_atomic_update_tx", + side_effect=lambda tr, key, cls, fn, idx: fn(node)): + block = db._claim_cntlid_block_tx(_NoTxnStore(store), b"node", b"alloc", node_id, + set(live), 4, ()) + return block, node, json.loads(kv[b"alloc"]) if b"alloc" in kv else None + + def test_smallest_free_block(self): + block, node, alloc = self._claim({"n1": 0, "n2": 2}) + self.assertEqual((block, node.cntlid_block), (1, 1)) + self.assertEqual(alloc, {"n1": 0, "n2": 2, "n3": 1}) + + def test_first_claim_and_idempotence(self): + self.assertEqual(self._claim(None)[0], 0) + block, _, alloc = self._claim({"n3": 2, "n1": 0}) + self.assertEqual((block, alloc), (2, {"n3": 2, "n1": 0})) + + def test_gone_nodes_release_their_block(self): + block, _, alloc = self._claim({"gone": 0, "n1": 1}, live=("n1", "n3")) + self.assertEqual(block, 0) + self.assertNotIn("gone", alloc) + + def test_full(self): + with self.assertRaisesRegex(ValueError, "at most 4"): + self._claim({"n1": 0, "n2": 1, "x": 2, "y": 3}, live=("n1", "n2", "n3", "x", "y")) + + def test_ensure_uses_an_existing_block(self): + node = StorageNode() + node.cntlid_block = 5 + db = MagicMock() + self.assertEqual(multi_attach.ensure_cntlid_block(node, db), 5) + db.claim_cntlid_block.assert_not_called() + node.cntlid_block = -1 + db.claim_cntlid_block.return_value = 7 + self.assertEqual(multi_attach.ensure_cntlid_block(node, db), 7) + self.assertEqual(node.cntlid_block, 7) + + +# ---------------------------------------------------------------- RPC + +class TestRpcWrappers(unittest.TestCase): + + def _params(self, call): + c = RPCClient("127.0.0.1", 8080, "u", cluster_ops.SecretStr("p")) + with patch.object(c, "_request", return_value=True) as req: + call(c) + return req.call_args.kwargs + + def test_subsystem_create_max_cntlid_only_when_given(self): + p = self._params(lambda c: c.subsystem_create("nqn", "sn", "mn")) + self.assertNotIn("max_cntlid", p) + self.assertEqual(p["min_cntlid"], 1) + p = self._params(lambda c: c.subsystem_create("nqn", "sn", "mn", min_cntlid=65, max_cntlid=128)) + self.assertEqual((p["min_cntlid"], p["max_cntlid"]), (65, 128)) + + def test_alceml_stuck_io_only_when_non_zero(self): + p = self._params(lambda c: c.bdev_alceml_create("a", "b", DEV_ID)) + self.assertNotIn("stuck_io_timeout_ms", p) + p = self._params(lambda c: c.bdev_alceml_create("a", "b", DEV_ID, stuck_io_timeout_ms=60000)) + self.assertEqual(p["stuck_io_timeout_ms"], 60000) + + +# ---------------------------------------------------------- device stacks + +def _stack_snode(): + snode = MagicMock() + snode.subsystem = f"{constants.CLUSTER_NQN}:host1_8080" + snode.enable_test_device = False + snode.cntlid_block = 2 + snode.nvmf_port = 4420 + iface = MagicMock(ip4_address="10.0.0.1", trtype="TCP", listen_address="10.0.0.1") + snode.data_nics = [iface] + return snode + + +class TestStorageDeviceStack(unittest.TestCase): + + def _run(self, dev, cluster): + snode = _stack_snode() + rpc = MagicMock() + rpc.bdev_get.return_value = {"aliases": ["pt-uuid"]} + with patch.object(storage_node_ops, "DBController") as db: + db.return_value.get_cluster_by_id.return_value = cluster + out = storage_node_ops._create_storage_device_stack(rpc, dev, snode, False) + self.assertIs(out, dev) + return snode, rpc + + def test_multi_attach_device(self): + dev = _device(multi=True) + snode, rpc = self._run(dev, _ma_cluster()) + nqn = f"{constants.CLUSTER_NQN}:lun:{DEV_ID}" + rpc.subsystem_create.assert_called_once_with(nqn, "sbcli-cn", DEV_ID, min_cntlid=129, max_cntlid=192) + rpc.nvmf_subsystem_add_ns.assert_called_once_with(nqn, f"alceml_{DEV_ID}_PT", + multi_attach.namespace_uuid(dev)) + self.assertEqual(snode.create_alceml.call_args.kwargs["stuck_io_timeout_ms"], 60000) + self.assertEqual(dev.nvmf_nqn, nqn) + + def test_regular_device_is_unchanged(self): + dev = _device() + cl = _ma_cluster() + cl.multi_attach = False + snode, rpc = self._run(dev, cl) + nqn = f"{constants.CLUSTER_NQN}:host1_8080:dev:{DEV_ID}" + rpc.subsystem_create.assert_called_once_with(nqn, "sbcli-cn", DEV_ID) + rpc.nvmf_subsystem_add_ns.assert_called_once_with(nqn, f"alceml_{DEV_ID}_PT", DEV_ID) + self.assertNotIn("stuck_io_timeout_ms", snode.create_alceml.call_args.kwargs) + + +class TestDeviceControllerStack(unittest.TestCase): + + def _run(self, dev, cluster): + from simplyblock_core.controllers import device_controller + snode = _stack_snode() + rpc = MagicMock() + rpc.bdev_get.return_value = None + rpc.subsystem_get.return_value = None + snode.rpc_client.return_value = rpc + with patch.object(device_controller, "DBController") as db: + db.return_value.get_cluster_by_id.return_value = cluster + self.assertTrue(device_controller._def_create_device_stack(dev, snode)) + return snode, rpc + + def test_multi_attach_device(self): + dev = _device(multi=True) + snode, rpc = self._run(dev, _ma_cluster()) + nqn = f"{constants.CLUSTER_NQN}:lun:{DEV_ID}" + rpc.subsystem_create.assert_called_once_with(nqn, "sbcli-cn", DEV_ID, min_cntlid=129, max_cntlid=192) + rpc.nvmf_subsystem_add_ns.assert_called_once_with(nqn, f"alceml_{DEV_ID}_PT", + multi_attach.namespace_uuid(dev)) + self.assertEqual(snode.create_alceml.call_args.kwargs["stuck_io_timeout_ms"], 60000) + self.assertEqual(dev.nvmf_nqn, nqn) + + def test_regular_device_is_unchanged(self): + dev = _device() + cl = _ma_cluster() + cl.multi_attach = False + snode, rpc = self._run(dev, cl) + nqn = f"{constants.CLUSTER_NQN}:host1_8080:dev:{DEV_ID}" + rpc.subsystem_create.assert_called_once_with(nqn, "sbcli-cn", DEV_ID) + rpc.nvmf_subsystem_add_ns.assert_called_once_with(nqn, f"alceml_{DEV_ID}_PT") + self.assertNotIn("stuck_io_timeout_ms", snode.create_alceml.call_args.kwargs) + + +# ---------------------------------------------------------------- attach + +class TestPeerAttach(unittest.TestCase): + + def _attach(self, dev): + dev.nvmf_ip = "10.0.0.5" + dev.nvmf_nqn = "nqn.x" + dev.nvmf_port = 4420 + target = MagicMock(active_rdma=False, active_tcp=True) + rpc = MagicMock() + rpc.bdev_nvme_controller_list.return_value = [] + rpc.bdev_get.return_value = {"name": "b"} + attach = MagicMock() + attach.bdev_nvme_attach_controller.return_value = [f"remote_{dev.alceml_bdev}n1"] + with patch.object(storage_node_ops, "DBController") as db: + db.return_value.get_storage_node_by_id.return_value = target + storage_node_ops._connect_device_attach( + f"remote_{dev.alceml_bdev}", dev, MagicMock(), rpc, attach, ["10.0.0.5"], False) + return attach.bdev_nvme_attach_controller.call_args + + def test_multi_attach_holds_io(self): + call = self._attach(_device(multi=True)) + self.assertEqual(call.kwargs["multipath"], "multipath") + self.assertEqual(call.kwargs["ctrlr_loss_timeout_sec"], constants.MULTI_ATTACH_HOLD_SEC) + self.assertEqual(call.kwargs["reconnect_delay_sec"], constants.MULTI_ATTACH_RECONNECT_DELAY_SEC) + self.assertEqual(call.kwargs["fast_io_fail_timeout_sec"], constants.MULTI_ATTACH_FAST_IO_FAIL_SEC) + + def test_regular_device_is_unchanged(self): + call = self._attach(_device()) + self.assertEqual(call.kwargs, {"multipath": False}) + + +# -------------------------------------------------------------- loopback + +def _map_node(uid, devices): + n = MagicMock(spec=StorageNode) + n.get_id.return_value = uid + n.cluster_id = "cluster-1" + n.status = StorageNode.STATUS_ONLINE + n.is_secondary_node = False + n.failure_domain = -1 + n.nvme_devices = devices + n.remote_devices = [] + return n + + +class TestOwnerLoopback(unittest.TestCase): + + def _map(self, node): + with patch.object(distr_controller, "DBController") as db: + cl = MagicMock(enable_node_affinity=False, enable_failure_domain=False, + sync_replication=False) + db.return_value.get_cluster_by_id.return_value = cl + return distr_controller.get_distr_cluster_map([node], node, "distr-1") + + def _remote(self, dev): + from simplyblock_core.models.nvme_device import RemoteDevice + rd = RemoteDevice() + rd.uuid = dev.uuid + rd.remote_bdev = f"remote_{dev.alceml_bdev}n1" + return rd + + def test_own_multi_attach_device_uses_the_loopback_bdev(self): + dev = _device(multi=True) + node = _map_node("node-a", [dev]) + node.remote_devices = [self._remote(dev)] + entry = self._map(node)["map_cluster"]["node-a"]["devices"][0] + self.assertEqual(entry["bdev_name"], f"remote_alceml_{DEV_ID}n1") + self.assertEqual(entry["status"], NVMeDevice.STATUS_ONLINE) + + def test_own_multi_attach_device_without_loopback_is_unavailable(self): + node = _map_node("node-a", [_device(multi=True)]) + entry = self._map(node)["map_cluster"]["node-a"]["devices"][0] + self.assertEqual(entry["status"], NVMeDevice.STATUS_UNAVAILABLE) + + def test_own_regular_device_uses_the_local_alceml(self): + node = _map_node("node-a", [_device()]) + entry = self._map(node)["map_cluster"]["node-a"]["devices"][0] + self.assertEqual(entry["bdev_name"], f"alceml_{DEV_ID}") + + def test_add_device_map_uses_the_loopback_bdev(self): + dev = _device(multi=True) + target = _map_node("node-a", [dev]) + target.rpc_client.return_value.distr_add_devices.return_value = True + with patch.object(distr_controller, "DBController") as db: + db.return_value.get_storage_node_by_id.return_value = target + with patch.object(distr_controller, "_sync_replication", return_value=False): + distr_controller.send_cluster_map_add_device(dev, target) + cl_map = target.rpc_client.return_value.distr_add_devices.call_args.args[0] + self.assertEqual(cl_map["devices"][0]["bdev_name"], f"remote_alceml_{DEV_ID}n1") + + def _connect(self, own_devices, peer_devices): + this = _map_node("node-a", own_devices) + peer = _map_node("node-b", peer_devices) + rpc = MagicMock() + rpc.bdev_get.side_effect = lambda name: {"name": name} if name.startswith("remote_") else None + this.rpc_client.return_value = rpc + connected = [] + with patch.object(storage_node_ops, "DBController") as db, \ + patch.object(storage_node_ops, "_connect_device_thread", + side_effect=lambda name, dev, node: connected.append(name)), \ + patch.object(storage_node_ops, "_fetch_bdev_name_set", return_value=set()): + db.return_value.get_storage_nodes_by_cluster_id.return_value = [this, peer] + out = storage_node_ops._connect_to_remote_devs(this) + return connected, {rd.get_id() for rd in out} + + def test_connect_includes_own_multi_attach_devices(self): + own = _device("own-1", multi=True, node_id="node-a") + peer = _device("peer-1", multi=True, node_id="node-b", order=1) + connected, ids = self._connect([own], [peer]) + self.assertEqual(sorted(connected), ["remote_alceml_own-1", "remote_alceml_peer-1"]) + self.assertEqual(ids, {"own-1", "peer-1"}) + + def test_connect_skips_own_regular_devices(self): + own = _device("own-1", node_id="node-a") + peer = _device("peer-1", node_id="node-b", order=1) + connected, ids = self._connect([own], [peer]) + self.assertEqual(connected, ["remote_alceml_peer-1"]) + self.assertEqual(ids, {"peer-1"}) + + +if __name__ == "__main__": + unittest.main() From 94574e6954db04a6fb1fb571648f73a774998cd6 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 12:31:03 +0100 Subject: [PATCH 08/50] feat(multi-attach): array LUN mode, phase 3 (LUN takeover by a standby) When the node serving a multi-attach LUN fails, the monitor starts a takeover per LUN (controllers/lun_takeover.take_over_lun) instead of marking the device unavailable: 1. fence the failed node at the array first (array plugin, host = node hostname); no plugin or no acknowledgement -> no takeover, event; 2. standby = owner's LVS secondary, else tertiary, else the owner itself; ONLINE, not fenced from the LUN; 3. the standby's agent (new POST /lun_info, docker + k8s) finds the LUN by WWN and reads block 0 for the alceml signature (read-only); 4. on the standby: aio base bdev, alceml OPEN-ONLY (pba_init_mode=1, never 2/3), PT, subsystem under the same host-independent NQN with the standby's cntlid range, same namespace UUID, standby listeners; partial exports are torn down (never formatted); 5. record serving_node_id / node_paths / nvmf_ip atomically (device stays ONLINE, no cluster-map change), then add the new path to remote_alceml_ on every ONLINE node (active_passive) and detach the dead node's path. Idempotent (re-run only completes paths), FDB lease + process lock per LUN, 90 s deadline inside the 120 s hold. No readable standby: hold, unavailable when the hold expires. Any other failure: unavailable. - simplyblock_core/array: ArrayPlugin (fence/unfence/capabilities), registry over entry-point group simplyblock.array_plugins, FakeArrayPlugin (tests; never registered by default); Cluster.array_plugin/array_endpoint/array_options, cluster_ops.set_array_plugin. NVMeDevice.fenced_node_ids. - monitor: set_node_offline, set_node_schedulable and the lost-site force-offline share _set_node_devices_unavailable; takeovers start right after the OFFLINE write, before (and in parallel with) the LVS ANA failover. LUNs of the node served elsewhere are left alone. - connect / repair: transport from the serving node; multi-attach controllers always inspected for missing paths, no wait on the dead path's transient state, active_passive policy. - owner return: restart builds no stack for LUNs served elsewhere or fenced from it, keeps those records (not REMOVED when unseen), does not flip their status; discovery and add-node never adopt (format) another node's LUN (matched by WWN/serial); remote-device connect / sync / reconnect include an offline owner's LUNs served by an ONLINE node; graceful shutdown leaves them alone; health check skips their local stack. No failback in this phase. - device_set_state: CAUSE_LUN_TAKEOVER allows ONLINE while the owner is down if the serving node is ONLINE; ONLINE refused while the serving node is down or fenced. - lun_takeover.unfence_node for an operator (never automatic). Regular clusters: every new path is gated on cluster/device multi_attach. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/array/__init__.py | 153 ++++ simplyblock_core/array/fake.py | 52 ++ simplyblock_core/cluster_ops.py | 28 + simplyblock_core/constants.py | 7 + .../controllers/device_controller.py | 40 +- .../controllers/events_controller.py | 3 + simplyblock_core/controllers/lun_takeover.py | 599 +++++++++++++++ simplyblock_core/db_controller.py | 44 ++ simplyblock_core/models/cluster.py | 9 + simplyblock_core/models/nvme_device.py | 4 + simplyblock_core/multi_attach.py | 135 ++++ .../services/health_check_service.py | 23 +- .../services/storage_node_monitor.py | 100 ++- simplyblock_core/snode_client.py | 6 + simplyblock_core/storage_node_ops.py | 159 +++- .../api/internal/storage_node/docker.py | 16 + .../api/internal/storage_node/kubernetes.py | 16 + simplyblock_web/node_utils.py | 37 + tests/unit/test_lun_takeover.py | 706 ++++++++++++++++++ 19 files changed, 2106 insertions(+), 31 deletions(-) create mode 100644 simplyblock_core/array/__init__.py create mode 100644 simplyblock_core/array/fake.py create mode 100644 simplyblock_core/controllers/lun_takeover.py create mode 100644 tests/unit/test_lun_takeover.py diff --git a/simplyblock_core/array/__init__.py b/simplyblock_core/array/__init__.py new file mode 100644 index 0000000000..35721d9d9e --- /dev/null +++ b/simplyblock_core/array/__init__.py @@ -0,0 +1,153 @@ +"""Storage-array integration of the multi-attach array LUN mode. + +A multi-attach cluster (``Cluster.multi_attach``) runs on array LUNs that are +presented to several storage nodes. Before a LUN is reopened on another node +the control plane must fence the node that served it at the array: that node's +host multipath layer may still hold queued writes, and once released they would +land after, and overwrite, newer data written through the new node. + +This package is the minimal interface the takeover needs (phase 3): an +abstract :class:`ArrayPlugin` with ``fence`` / ``unfence`` / ``capabilities``, +and a registry that loads a vendor plugin by name from the entry-point group +``simplyblock.array_plugins``. Credentials (KMS) and further operations are +phase 4. + +Plugins are found only through the entry-point group or an explicit +:func:`register_plugin` call. The test double in :mod:`.fake` is never +registered by default, so a cluster cannot be configured with a plugin that +fences nothing. +""" + +from __future__ import annotations + +import abc +import logging +import threading +from importlib import metadata + +logger = logging.getLogger() + +#: The entry-point group vendor plugins register under; the entry point's +#: name is the value of ``Cluster.array_plugin``. +ENTRY_POINT_GROUP = "simplyblock.array_plugins" + + +class ArrayError(Exception): + """An array operation failed, or no plugin could be loaded.""" + + +class ArrayNotConfigured(ArrayError): + """The cluster names no array plugin.""" + + +class ArrayPlugin(abc.ABC): + """The array operations the control plane needs. + + ``lun_wwn`` is the LUN's world-wide name as the hosts report it + (``NVMeDevice.lun_wwn``); ``host`` is the storage node's identity on the + array (see ``multi_attach.array_host``). Every operation is idempotent: a + fence of a host that is already fenced succeeds. A method reports failure + by raising (any exception, preferably :class:`ArrayError`) or returning a + falsy value; the caller treats both the same way. + """ + + def __init__(self, endpoint: str = "", options: dict | None = None, credentials=None): + self.endpoint = endpoint + self.options = dict(options or {}) + # Phase 4 resolves these from the KMS; a placeholder until then. + self.credentials = credentials + + @abc.abstractmethod + def fence(self, lun_wwn: str, host: str) -> bool: + """Revoke ``host``'s access to the LUN so no I/O of it, including I/O + queued in its multipath layer, can reach the LUN any more. Returns + only once the array has applied it.""" + + @abc.abstractmethod + def unfence(self, lun_wwn: str, host: str) -> bool: + """Give ``host`` access to the LUN again.""" + + def capabilities(self) -> set[str]: + """Names of the optional operations the plugin supports (phase 4).""" + return {"fence", "unfence"} + + +_registry: dict[str, type[ArrayPlugin]] = {} +_registry_lock = threading.Lock() + + +def register_plugin(name: str, cls: type[ArrayPlugin]) -> None: + """Register ``cls`` under ``name`` in this process (tests; built-in + plugins). An entry point of the same name is shadowed.""" + if not (isinstance(cls, type) and issubclass(cls, ArrayPlugin)): + raise TypeError(f"{cls!r} is not an ArrayPlugin") + with _registry_lock: + _registry[name] = cls + + +def unregister_plugin(name: str) -> None: + with _registry_lock: + _registry.pop(name, None) + + +def _entry_point(name: str): + eps = metadata.entry_points() + if hasattr(eps, "select"): + found = list(eps.select(group=ENTRY_POINT_GROUP, name=name)) + else: # pragma: no cover - Python < 3.10 + found = [ep for ep in eps.get(ENTRY_POINT_GROUP, []) if ep.name == name] + return found[0] if found else None + + +def plugin_class(name: str) -> type[ArrayPlugin]: + """The plugin class registered as ``name``. + + Raises: + ArrayError: no plugin of that name, or it is not an ArrayPlugin. + """ + with _registry_lock: + cls = _registry.get(name) + if cls is not None: + return cls + ep = _entry_point(name) + if ep is None: + raise ArrayError(f"no array plugin {name!r} in entry-point group {ENTRY_POINT_GROUP}") + try: + cls = ep.load() + except Exception as e: + raise ArrayError(f"array plugin {name!r} failed to load: {e}") from e + if not (isinstance(cls, type) and issubclass(cls, ArrayPlugin)): + raise ArrayError(f"array plugin {name!r} ({ep.value}) is not an ArrayPlugin") + return cls + + +def load_plugin(name: str, endpoint: str = "", options: dict | None = None, + credentials=None) -> ArrayPlugin: + """An instance of plugin ``name`` for ``endpoint``. + + Raises: + ArrayNotConfigured: ``name`` is empty. + ArrayError: the plugin cannot be found, loaded or constructed. + """ + if not name: + raise ArrayNotConfigured("no array plugin is configured") + cls = plugin_class(name) + try: + return cls(endpoint=endpoint, options=options, credentials=credentials) + except Exception as e: + raise ArrayError(f"array plugin {name!r} could not be set up: {e}") from e + + +def plugin_for_cluster(cluster) -> ArrayPlugin: + """The array plugin a cluster is configured with (``Cluster.array_plugin`` + / ``array_endpoint`` / ``array_options``). + + Raises: + ArrayNotConfigured: the cluster names no plugin. + ArrayError: it cannot be loaded. + """ + return load_plugin( + getattr(cluster, "array_plugin", "") or "", + getattr(cluster, "array_endpoint", "") or "", + getattr(cluster, "array_options", None) or {}, + ) diff --git a/simplyblock_core/array/fake.py b/simplyblock_core/array/fake.py new file mode 100644 index 0000000000..1d30916e21 --- /dev/null +++ b/simplyblock_core/array/fake.py @@ -0,0 +1,52 @@ +"""In-memory array plugin for tests. Never registered by default.""" + +from __future__ import annotations + +import threading +from typing import ClassVar + +from simplyblock_core.array import ArrayError, ArrayPlugin + + +class FakeArrayPlugin(ArrayPlugin): + """Records every call in ``calls`` (``(op, lun_wwn, host)``) and the + fenced pairs in ``fenced``. ``fail`` = a set of operation names + (``"fence"``, ``"unfence"``) that raise :class:`ArrayError`; + ``fail_falsy`` the ones that return False instead. + + State is per class (``reset()`` clears it), because the takeover builds a + fresh instance from the cluster record each time.""" + + calls: ClassVar[list] = [] + fenced: ClassVar[set] = set() + fail: ClassVar[set] = set() + fail_falsy: ClassVar[set] = set() + _lock = threading.Lock() + + @classmethod + def reset(cls): + with cls._lock: + cls.calls = [] + cls.fenced = set() + cls.fail = set() + cls.fail_falsy = set() + + def _op(self, op, lun_wwn, host): + with self._lock: + type(self).calls.append((op, lun_wwn, host)) + if op in self.fail: + raise ArrayError(f"fake array: {op} of {host} on {lun_wwn} failed") + if op in self.fail_falsy: + return False + with self._lock: + if op == "fence": + type(self).fenced.add((lun_wwn, host)) + else: + type(self).fenced.discard((lun_wwn, host)) + return True + + def fence(self, lun_wwn, host): + return self._op("fence", lun_wwn, host) + + def unfence(self, lun_wwn, host): + return self._op("unfence", lun_wwn, host) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 49fbb2be85..2900d4aa13 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -1024,6 +1024,34 @@ def mutate(fresh): db_controller.atomic_update(cluster, mutate) +def set_array_plugin(cl_id, plugin: str, endpoint: str = "", options: dict | None = None) -> None: + """Configure the storage-array plugin of a multi-attach cluster (the + plugin that fences a failed node from its LUNs before a takeover; see + simplyblock_core.array). An empty ``plugin`` clears it, after which + takeovers are refused. Credentials come from the KMS (later phase). + + Raises: + ValueError: not a multi-attach cluster, or the plugin cannot be loaded. + """ + from simplyblock_core import array, multi_attach + cluster = db_controller.get_cluster_by_id(cl_id) + if not multi_attach.is_multi_attach(cluster): + raise ValueError("an array plugin is only used by a multi-attach cluster") + if plugin: + try: + array.plugin_class(plugin) + except array.ArrayError as e: + raise ValueError(str(e)) from e + opts = dict(options or {}) + + def mutate(fresh): + fresh.array_plugin = plugin or "" + fresh.array_endpoint = endpoint or "" + fresh.array_options = opts + + db_controller.atomic_update(cluster, mutate) + + def _wait_for_full_device_connectivity(cl_id, timeout_sec=300, poll_sec=10, checkpoint=None): """Block until every ONLINE primary node holds a connected remote-device record for every ONLINE device of every OTHER online node, or raise. diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 37a3771b8d..f34e4054da 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -1068,6 +1068,13 @@ def get_config_var(name, default=None): MULTI_ATTACH_CNTLID_BLOCK_SIZE = 64 MULTI_ATTACH_MAX_CNTLID = 0xFFEF MULTI_ATTACH_MAX_NODES = 1022 +# A LUN takeover (controllers/lun_takeover) must finish well inside the +# peers' hold, or their I/O fails before the new path exists; it gives up at +# this deadline (seconds from its start) and the device goes unavailable. +MULTI_ATTACH_TAKEOVER_DEADLINE_SEC = 90 +# Lease of a running takeover (one per LUN, in FDB): a holder older than this +# is presumed dead. +MULTI_ATTACH_TAKEOVER_LOCK_TTL_SEC = 2 * MULTI_ATTACH_TAKEOVER_DEADLINE_SEC MIG_JOB_SIZE = 64 # Sync-replication resync (tasks_runner_sync_resync): the shared task driver diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index d5db4be013..977e7adaf2 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -130,6 +130,10 @@ def get_storage_node_by_jm_device(db_controller: DBController, id) -> StorageNod # when IO resumes (2026-07-06 failback incident: placement failures because # peers still saw the recovering node's devices UNAVAILABLE). CAUSE_NODE_RECOVERY = "node_recovery" +# Multi-attach array LUN mode: the device's LUN is served by another node +# (controllers/lun_takeover) while its owner is not ONLINE. The device may go +# ONLINE as long as its SERVING node is ONLINE. +CAUSE_LUN_TAKEOVER = "lun_takeover" def _atomic_device_set(db_controller, node_id, device_id, apply_fields): @@ -202,6 +206,22 @@ def _connect_peer(peer_id): th.join() +def _serving_node_online(db_controller, device): + try: + server = db_controller.get_storage_node_by_id(device.serving_node()) + except KeyError: + return False + return server.status == StorageNode.STATUS_ONLINE + + +def _served_by_online_node(db_controller, device, cause): + """CAUSE_LUN_TAKEOVER only: the multi-attach device is served by another + node (serving_node_id) and that node is ONLINE.""" + if cause != CAUSE_LUN_TAKEOVER or not multi_attach.served_elsewhere(device, device.node_id): + return False + return _serving_node_online(db_controller, device) + + def device_set_state(device_id, state, cause=CAUSE_OTHER, connect_peers=True): db_controller = DBController() try: @@ -238,13 +258,31 @@ def device_set_state(device_id, state, cause=CAUSE_OTHER, connect_peers=True): # ONLINE node, so it is unaffected. if (state == NVMeDevice.STATUS_ONLINE and snode.status != StorageNode.STATUS_ONLINE - and cause not in (CAUSE_DEVICE_RESTART, CAUSE_NODE_RECOVERY)): + and cause not in (CAUSE_DEVICE_RESTART, CAUSE_NODE_RECOVERY) + and not _served_by_online_node(db_controller, device, cause)): logger.warning( f"Refusing to set device {device_id} ONLINE while node " f"{snode.get_id()} is {snode.status} (cause={cause}); " f"node bring-up owns device re-online") return False + # Multi-attach: a device whose LUN another node serves is ONLINE only + # while that node is; its owner's recovery does not bring it back (the + # owner built nothing for it). + if (state == NVMeDevice.STATUS_ONLINE + and multi_attach.served_elsewhere(device, snode.get_id()) + and not _serving_node_online(db_controller, device)): + logger.warning( + f"Refusing to set device {device_id} ONLINE: its LUN is served by node " + f"{device.serving_node()}, which is not ONLINE (cause={cause})") + return False + if (state == NVMeDevice.STATUS_ONLINE + and multi_attach.fenced_from(device, device.serving_node())): + logger.warning( + f"Refusing to set device {device_id} ONLINE: node {device.serving_node()} " + f"is fenced from its LUN (cause={cause})") + return False + # Failed is a terminal state. The only allowed exits are: # STATUS_FAILED → STATUS_FAILED_AND_MIGRATED (failure data migration) # STATUS_FAILED → STATUS_ONLINE (explicit device restart) diff --git a/simplyblock_core/controllers/events_controller.py b/simplyblock_core/controllers/events_controller.py index 611ddbf4e0..9d1764f8c0 100644 --- a/simplyblock_core/controllers/events_controller.py +++ b/simplyblock_core/controllers/events_controller.py @@ -22,6 +22,9 @@ #: A client LVS port was reopened FENCE_RELEASE_BUDGET times in one window #: and SPDK fenced it again each time; the control plane stopped reopening it. EVENT_FENCE_LOOP = "FENCE_LOOP" +#: Multi-attach array LUN mode: a LUN was (or could not be) taken over by +#: another node after its serving node failed. +EVENT_LUN_TAKEOVER = "LUN_TAKEOVER" DOMAIN_CLUSTER = "cluster" DOMAIN_MANAGEMENT = "management" diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py new file mode 100644 index 0000000000..25bb87b231 --- /dev/null +++ b/simplyblock_core/controllers/lun_takeover.py @@ -0,0 +1,599 @@ +"""Multi-attach array LUN mode, phase 3: take a LUN over on a standby node. + +When the node serving a multi-attach device (``NVMeDevice.serving_node()``) +fails, its peers' controllers ``remote_alceml_`` hold I/O for +``MULTI_ATTACH_HOLD_SEC``. Within that window :func:`take_over_lun`: + +1. fences the failed node from the LUN at the array -- before anything else: + its host multipath layer may still hold queued writes that would land late + and overwrite newer data; +2. picks a standby: the owner's LVS secondary, else its tertiary, else the + owner itself; ONLINE and not fenced from this LUN; +3. verifies through the standby's node agent that it sees the LUN and that + block 0 carries the alceml signature (read-only); +4. on the standby builds the base bdev, the alceml OPEN-ONLY + (``pba_init_mode=1``; 2/3 would format the LUN and destroy its data), the + PT bdev and the device subsystem under the same NQN, with the standby's + controller-ID range, the same namespace UUID and the standby's listeners; +5. records the standby as the serving node (atomically; the device stays + ONLINE, no unavailable broadcast, no cluster-map change); +6. adds the standby's export as a path of ``remote_alceml_`` on every + ONLINE node (the standby too: its loopback), where the held I/O resumes; +7. detaches the failed node's path from those controllers so it never + reconnects. + +The record is written before the paths are added (design order: paths, then +record): the export on the standby is what serves the LUN from then on, and +any node that misses its path in step 6 repairs it from the record +(connect_device uses the serving node). A re-run is idempotent: each step +checks what is already in place, and a LUN already served by an ONLINE node +other than the failed one only gets steps 6-7. + +Never done: a takeover without a configured array plugin or without the +array's acknowledgement of the fence; an alceml create with any init mode but +open-only. When no standby can read the LUN, nothing is changed: peers keep +holding until their hold expires, and the caller marks the device +unavailable. +""" + +from __future__ import annotations + +import logging +import os +import socket +import threading +import time +import uuid as uuid_mod + +from simplyblock_core import array, constants, multi_attach, utils +from simplyblock_core.controllers import events_controller as ec +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.events import EventObj +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + +logger = logging.getLogger() + +#: bdev_alceml_create pba_init_mode that opens an existing alceml without +#: touching its data. The only mode a takeover ever uses. +ALCEML_OPEN_ONLY = 1 + +#: Device states a takeover acts on. A failed, removed or new device is not +#: taken over. +_TAKEOVER_STATES = (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, + NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE) + +_local_locks: dict[str, threading.Lock] = {} +_local_locks_guard = threading.Lock() + + +#: Outcomes of :func:`take_over_lun_outcome`. +TAKEN_OVER = "taken_over" # served by a live node other than the failed one +IN_PROGRESS = "in_progress" # another takeover of this LUN is running +NO_STANDBY = "no_standby" # fenced, but no standby can read the LUN: hold +FAILED = "failed" # not taken over (no plugin, fence, export, ...) + + +class TakeoverError(Exception): + """A takeover step failed; the message says which and why.""" + + +def _local_lock(device_id) -> threading.Lock: + with _local_locks_guard: + return _local_locks.setdefault(device_id, threading.Lock()) + + +def _event(cluster_id, device, message, level=EventObj.LEVEL_INFO): + try: + ec.log_event_cluster( + cluster_id=cluster_id, + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_LUN_TAKEOVER, + db_object=device, + caused_by=ec.CAUSED_BY_MONITOR, + message=message, + node_id=device.get_id(), + event_level=level, + storage_id=device.cluster_device_order) + except Exception as e: # noqa: BLE001 + logger.error("LUN takeover event of %s not logged: %s", device.get_id(), e) + + +def _check_deadline(deadline, step): + if time.monotonic() > deadline: + raise TakeoverError( + f"deadline of {constants.MULTI_ATTACH_TAKEOVER_DEADLINE_SEC}s passed before {step}") + + +def _remaining(deadline, cap): + return max(1, min(cap, int(deadline - time.monotonic()))) + + +def _transport(node): + if node.active_rdma: + return "RDMA" + if node.active_tcp: + return "TCP" + return None + + +def _device_on_owner(owner, device_id): + for dev in owner.nvme_devices: + if dev.get_id() == device_id: + return dev + return None + + +def take_over_lun(device_id, failed_node_id, reason="", *, deadline_sec=None) -> bool: + """Move the LUN of multi-attach device ``device_id`` off the failed node + ``failed_node_id`` onto a standby (see the module docstring). + + Returns True when the device is served by an ONLINE node other than the + failed one afterwards (taken over now, or already before), False when it + was not taken over (the reason is logged and raised as a cluster event) + or another takeover of it is running. Never raises. + """ + return take_over_lun_outcome(device_id, failed_node_id, reason, + deadline_sec=deadline_sec) == TAKEN_OVER + + +def take_over_lun_outcome(device_id, failed_node_id, reason="", *, deadline_sec=None) -> str: + """:func:`take_over_lun`, answering one of TAKEN_OVER, IN_PROGRESS, + NO_STANDBY or FAILED.""" + db = DBController() + deadline = time.monotonic() + (deadline_sec if deadline_sec is not None + else constants.MULTI_ATTACH_TAKEOVER_DEADLINE_SEC) + try: + dev = db.get_storage_device_by_id(device_id) + owner = db.get_storage_node_by_id(dev.node_id) + cluster = db.get_cluster_by_id(owner.cluster_id) + except KeyError as e: + logger.error("LUN takeover of %s: %s", device_id, e) + return FAILED + if not (multi_attach.is_multi_attach(cluster) and multi_attach.is_multi_attach(dev)): + logger.error("LUN takeover of %s refused: not a multi-attach device", device_id) + return FAILED + + local = _local_lock(device_id) + if not local.acquire(blocking=False): + logger.info("LUN takeover of %s already running in this process", device_id) + return IN_PROGRESS + token = f"{socket.gethostname()}:{os.getpid()}:{uuid_mod.uuid4().hex[:8]}" + try: + try: + held, holder = db.acquire_lun_takeover_lock( + device_id, token, constants.MULTI_ATTACH_TAKEOVER_LOCK_TTL_SEC) + except Exception as e: # noqa: BLE001 + logger.error("LUN takeover of %s: lease not taken: %s", device_id, e) + return FAILED + if not held: + logger.info("LUN takeover of %s already running (%s)", device_id, holder) + return IN_PROGRESS + try: + return _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) + finally: + try: + db.release_lun_takeover_lock(device_id, token) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover of %s: lease not released: %s", device_id, e) + finally: + local.release() + + +def _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) -> str: + owner = db.get_storage_node_by_id(db.get_storage_device_by_id(device_id).node_id) + dev = _device_on_owner(owner, device_id) + if dev is None: + logger.error("LUN takeover of %s: device not on its owner %s", device_id, owner.get_id()) + return FAILED + cluster_id = owner.cluster_id + logger.info("LUN takeover of %s (LUN %s, owner %s) from node %s: %s", + device_id, dev.lun_wwn, owner.get_id(), failed_node_id, reason or "node lost") + + if dev.status not in _TAKEOVER_STATES: + logger.info("LUN takeover of %s skipped: device is %s", device_id, dev.status) + return FAILED + + nodes_by_id = {n.get_id(): n for n in db.get_storage_nodes_by_cluster_id(cluster_id)} + serving_id = dev.serving_node() + if serving_id != failed_node_id: + # Re-run, or a stale trigger: only complete the paths if the LUN is + # served by a live node. + server = nodes_by_id.get(serving_id) + if server is None or server.status != StorageNode.STATUS_ONLINE: + logger.warning("LUN takeover of %s: served by %s (%s), not by %s; nothing done", + device_id, serving_id, getattr(server, "status", "unknown"), failed_node_id) + return FAILED + logger.info("LUN takeover of %s: already served by %s; completing its paths", + device_id, serving_id) + failed = nodes_by_id.get(failed_node_id) + _switch_paths(db, cluster_id, dev, server, failed, _failed_paths(failed, None, None)) + return TAKEN_OVER + + failed = nodes_by_id.get(failed_node_id) + if failed is None: + logger.error("LUN takeover of %s: failed node %s not found", device_id, failed_node_id) + return FAILED + old_ips = [ip.strip() for ip in (dev.nvmf_ip or "").split(",") if ip.strip()] + old_port = dev.nvmf_port + + try: + # 1. Fence -- before anything else. + _fence(cluster, dev, failed) + _record_fenced(db, owner, device_id, failed_node_id) + _check_deadline(deadline, "choosing a standby") + + # 2 + 3. A standby that can read the LUN. + standby, path, why = _choose_standby(owner, dev, failed_node_id, nodes_by_id, deadline) + if standby is None: + msg = (f"LUN {dev.lun_wwn} of device {device_id} not taken over from node " + f"{failed_node_id}: no standby can read it ({why}); holding I/O until " + f"the peers' hold expires") + logger.error(msg) + _event(cluster_id, dev, msg, EventObj.LEVEL_CRITICAL) + return NO_STANDBY + _check_deadline(deadline, f"exporting the LUN on {standby.get_id()}") + + # 4. Open-only export on the standby. + export = _export_on_standby(db, cluster, standby, dev, path, deadline) + _check_deadline(deadline, "recording the takeover") + + # 5 (design step 6). Record, atomically. + if not _record_serving(db, owner, device_id, failed_node_id, standby, path, export): + raise TakeoverError("the device record changed under the takeover") + except TakeoverError as e: + msg = f"LUN takeover of device {device_id} from node {failed_node_id} failed: {e}" + logger.error(msg) + _event(cluster_id, dev, msg, EventObj.LEVEL_CRITICAL) + return FAILED + except Exception as e: # noqa: BLE001 + msg = f"LUN takeover of device {device_id} from node {failed_node_id} failed: {e}" + logger.exception(msg) + _event(cluster_id, dev, msg, EventObj.LEVEL_CRITICAL) + return FAILED + + dev = _device_on_owner(db.get_storage_node_by_id(owner.get_id()), device_id) or dev + # 6 + 7 (design steps 5 and 7). New path on every ONLINE node, old one off. + added = _switch_paths(db, cluster_id, dev, standby, failed, + _failed_paths(failed, old_ips, old_port)) + + if dev.status != NVMeDevice.STATUS_ONLINE: + # The hold had expired and the device was marked unavailable; it is + # served again. + from simplyblock_core.controllers import device_controller + device_controller.device_set_state( + device_id, NVMeDevice.STATUS_ONLINE, + cause=device_controller.CAUSE_LUN_TAKEOVER, connect_peers=False) + + msg = (f"LUN {dev.lun_wwn} of device {device_id} taken over from node {failed_node_id} " + f"by node {standby.get_id()} ({path}); new path on {len(added)} node(s)") + logger.info(msg) + _event(cluster_id, dev, msg, EventObj.LEVEL_WARN) + return TAKEN_OVER + + +# ---------------------------------------------------------------- 1. fence + +def _fence(cluster, dev, failed): + host = multi_attach.array_host(failed) + if not host: + raise TakeoverError(f"node {failed.get_id()} has no hostname to fence it by") + if not dev.lun_wwn: + raise TakeoverError("the device has no LUN WWN to fence") + try: + plugin = array.plugin_for_cluster(cluster) + except array.ArrayNotConfigured: + raise TakeoverError( + "no array plugin is configured (cluster array_plugin); a LUN is never " + "taken over without fencing its serving node") + except array.ArrayError as e: + raise TakeoverError(f"array plugin unavailable: {e}") + try: + ok = plugin.fence(dev.lun_wwn, host) + except Exception as e: # noqa: BLE001 + raise TakeoverError(f"array fence of {host} on LUN {dev.lun_wwn} failed: {e}") + if not ok: + raise TakeoverError(f"array did not acknowledge the fence of {host} on LUN {dev.lun_wwn}") + logger.info("LUN %s: node %s (%s) fenced at the array", dev.lun_wwn, failed.get_id(), host) + + +def _record_fenced(db, owner, device_id, node_id): + def _mut(n): + d = _device_on_owner(n, device_id) + if d is None or node_id in d.fenced_node_ids: + return False + d.fenced_node_ids = list(d.fenced_node_ids) + [node_id] + return True + db.atomic_update(owner, _mut) + + +# --------------------------------------------------------- 2 + 3. standby + +def _choose_standby(owner, dev, failed_node_id, nodes_by_id, deadline): + reasons = [] + for nid in multi_attach.standby_candidates(owner, failed_node_id): + node = nodes_by_id.get(nid) + if node is None: + continue + if node.status != StorageNode.STATUS_ONLINE: + reasons.append(f"{nid}: {node.status}") + continue + if nid in (dev.fenced_node_ids or []): + reasons.append(f"{nid}: fenced from this LUN") + continue + if _transport(node) is None: + reasons.append(f"{nid}: no active fabric") + continue + path, why = _probe(node, dev, deadline) + if path: + logger.info("LUN %s: standby %s reads it at %s", dev.lun_wwn, nid, path) + return node, path, "" + reasons.append(f"{nid}: {why}") + if time.monotonic() > deadline: + reasons.append("deadline passed") + break + return None, None, "; ".join(reasons) or "no standby candidate" + + +def _probe(node, dev, deadline): + try: + result, err = node.client(timeout=_remaining(deadline, 15)).lun_info(dev.lun_wwn) + except Exception as e: # noqa: BLE001 + return None, f"agent query failed: {e}" + if err: + return None, f"agent: {err}" + if not isinstance(result, list): + return None, "agent returned no LUN inventory" + return multi_attach.pick_lun_path(result, dev) + + +# --------------------------------------------------------------- 4. export + +def _create_base_bdev(rpc, name, path, dev): + """The base bdev over the LUN, with the engine and block size the device + records when it records them (lblk I/O-engine fields, if present).""" + engine = getattr(dev, "lblk_io_engine", "") or "aio" + block_size = int(getattr(dev, "aio_block_size", 0) or 0) + if engine == "uring": + if hasattr(rpc, "bdev_uring_create"): + return rpc.bdev_uring_create(name, path, block_size) if block_size else rpc.bdev_uring_create(name, path) + params = {"name": name, "filename": path} + if block_size: + params["block_size"] = block_size + return rpc._request("bdev_uring_create", **params) + return rpc.bdev_aio_create(name, path, block_size) + + +def _export_on_standby(db, cluster, standby, dev, path, deadline) -> dict: + if getattr(dev, "bdev_type", "aio") != "aio": + raise TakeoverError(f"device type {dev.bdev_type!r} cannot be taken over (array LUNs are lblk)") + rpc = standby.rpc_client(timeout=_remaining(deadline, 60)) + sid = standby.get_id() + created = [] + try: + base = utils.aio_bdev_name_for_serial(dev.serial_number) + info = rpc.bdev_get(base) + if not info: + if not _create_base_bdev(rpc, base, path, dev): + raise TakeoverError(f"base bdev {base} ({path}) not created on {sid}") + created.append(("base", base)) + info = rpc.bdev_get(base) + if not info: + raise TakeoverError(f"base bdev {base} not found on {sid} after create") + size = int(info.get("block_size", 0)) * int(info.get("num_blocks", 0)) + if size < int(dev.size or 0): + raise TakeoverError(f"base bdev {base} on {sid} is {size} bytes, the device {dev.size}") + + alceml = dev.alceml_bdev or f"alceml_{dev.get_id()}" + if not rpc.bdev_get(alceml): + checksum_method, cache_size, cache_eviction = utils.alceml_checksum_params(cluster, dev) + ok = standby.create_alceml( + alceml, base, dev.get_id(), + pba_init_mode=ALCEML_OPEN_ONLY, + write_protection=cluster.distr_ndcs > 1, + pba_page_size=cluster.page_size_in_blocks, + full_page_unmap=cluster.full_page_unmap, + checksum_method=checksum_method, + cache_size=cache_size, + cache_eviction_threshold=cache_eviction, + force_4k_atomic=getattr(cluster, 'atomic_4k', False), + **multi_attach.alceml_create_kwargs(dev), + ) + if not ok: + raise TakeoverError(f"alceml {alceml} not opened on {sid}") + created.append(("alceml", alceml)) + try: + from simplyblock_core.controllers import qos_controller + rpc.alceml_set_qos_weights(qos_controller.get_qos_weights_list(cluster.get_id())) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: QoS weights not set on %s: %s", sid, e) + + pt = dev.pt_bdev or f"{alceml}_PT" + pt_info = rpc.bdev_get(pt) + if not pt_info: + if not rpc.bdev_PT_NoExcl_create(pt, alceml): + raise TakeoverError(f"PT bdev {pt} not created on {sid}") + created.append(("pt", pt)) + pt_info = rpc.bdev_get(pt) + if not pt_info: + raise TakeoverError(f"PT bdev {pt} not found on {sid} after create") + + nqn = multi_attach.device_subsystem_nqn(standby, dev, cluster) + subsys = rpc.subsystem_get(nqn) + if not subsys: + if not rpc.subsystem_create(nqn, 'sbcli-cn', dev.get_id(), + **multi_attach.subsystem_create_kwargs(standby, dev, db)): + raise TakeoverError(f"subsystem {nqn} not created on {sid}") + created.append(("subsystem", nqn)) + subsys = {} + listening = {(la or {}).get("traddr") for la in (subsys.get("listen_addresses") or [])} + ips = [] + for iface in standby.data_nics: + if not iface.ip4_address: + continue + if iface.ip4_address not in listening: + rpc.listeners_create(nqn, iface.trtype, iface.listen_address, standby.nvmf_port) + ips.append(iface.ip4_address) + if not ips: + raise TakeoverError(f"node {sid} has no data IP to export the LUN on") + if not any(ns.get("name") == pt for ns in (subsys.get("namespaces") or [])): + if not rpc.nvmf_subsystem_add_ns(nqn, pt, multi_attach.namespace_uuid(dev)): + raise TakeoverError(f"namespace {pt} not added to {nqn} on {sid}") + except Exception: + _teardown(rpc, sid, created) + raise + aliases = (pt_info or {}).get("aliases") or [] + return { + "nqn": nqn, + "ips": ips, + "port": standby.nvmf_port, + "base": base, + "pt_uuid": aliases[0] if aliases else "", + } + + +def _teardown(rpc, sid, created): + """Undo what this attempt created on the standby, newest first, so a + failed takeover leaves no alceml open on a LUN the node does not serve. + Never formats anything; best-effort.""" + for kind, name in reversed(created): + try: + if kind == "subsystem": + rpc.subsystem_delete(name) + elif kind == "pt": + rpc.bdev_PT_NoExcl_delete(name) + elif kind == "alceml": + rpc.bdev_alceml_delete(name) + elif kind == "base": + rpc.bdev_aio_delete(name) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover cleanup on %s: %s %s not removed: %s", sid, kind, name, e) + + +# --------------------------------------------------------------- 5. record + +def _record_serving(db, owner, device_id, failed_node_id, standby, path, export) -> bool: + standby_id = standby.get_id() + ips = export["ips"] + + def _mut(n): + d = _device_on_owner(n, device_id) + if d is None or d.serving_node() not in (failed_node_id, standby_id): + return False + d.serving_node_id = "" if standby_id == d.node_id else standby_id + paths = dict(d.node_paths or {}) + paths[standby_id] = {"path": path, "bdev": export["base"]} + d.node_paths = paths + d.nvmf_nqn = export["nqn"] + d.nvmf_ip = ",".join(ips) + d.nvmf_port = export["port"] + d.nvmf_multipath = len(ips) > 1 + if export["pt_uuid"]: + d.pt_spdk_uuid = export["pt_uuid"] + return True + + fresh = db.atomic_update(owner, _mut) + d = _device_on_owner(fresh, device_id) if fresh is not None else None + return d is not None and d.serving_node() == standby_id + + +# ---------------------------------------------------------- 6 + 7. paths + +def _failed_paths(failed, old_ips, old_port): + """``(ips, port, trtype)`` of the failed node's export.""" + if failed is None: + return [], None, "TCP" + ips = list(old_ips or []) + for iface in failed.data_nics or []: + if iface.ip4_address and iface.ip4_address not in ips: + ips.append(iface.ip4_address) + return ips, old_port or failed.nvmf_port, _transport(failed) or "TCP" + + +def _switch_paths(db, cluster_id, dev, standby, failed, failed_paths) -> list: + """Add the standby's export as a path of ``remote_alceml_`` on every + ONLINE node, then detach the failed node's path there. Returns the ids + of the nodes that have the new path.""" + name = f"remote_{dev.alceml_bdev}" + trtype = _transport(standby) or "TCP" + new_ips = [ip.strip() for ip in (dev.nvmf_ip or "").split(",") if ip.strip()] + dead_ips, dead_port, dead_trtype = failed_paths + dead_ips = [ip for ip in dead_ips if ip not in new_ips] + failed_id = failed.get_id() if failed is not None else None + added = [] + for node in db.get_storage_nodes_by_cluster_id(cluster_id): + if node.status != StorageNode.STATUS_ONLINE or node.get_id() == failed_id: + continue + try: + rpc = node.rpc_client(timeout=5, retry=1) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: no RPC client for %s: %s", node.get_id(), e) + continue + ok = False + for ip in new_ips: + try: + if rpc.bdev_nvme_attach_controller( + name, dev.nvmf_nqn, ip, dev.nvmf_port, trtype, multipath="multipath", + **multi_attach.attach_controller_kwargs(dev)): + ok = True + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: path %s of %s not added on %s: %s", + ip, name, node.get_id(), e) + if not ok: + logger.error("LUN takeover: %s on %s has no path to the new export; the health " + "check reconnects it from the record", name, node.get_id()) + continue + added.append(node.get_id()) + try: + rpc.bdev_nvme_set_multipath_policy(f"{name}n1", "active_passive") + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: multipath policy of %sn1 on %s not set: %s", + name, node.get_id(), e) + for ip in dead_ips: + try: + rpc.bdev_nvme_detach_controller(name, traddr=ip, trsvcid=dead_port, + trtype=dead_trtype) + except Exception as e: # noqa: BLE001 + # Usually: the path is already gone (the hold destructed it). + logger.info("LUN takeover: old path %s of %s on %s not detached: %s", + ip, name, node.get_id(), e) + return added + + +# ------------------------------------------------------------- operator + +def unfence_node(device_id, node_id) -> bool: + """Give ``node_id`` access to the LUN of ``device_id`` again and drop it + from ``fenced_node_ids``. Never called automatically: only once the + node's host is known to hold no queued writes for the LUN (its multipath + queues drained or the host rebooted). Returns True on success.""" + db = DBController() + try: + dev = db.get_storage_device_by_id(device_id) + owner = db.get_storage_node_by_id(dev.node_id) + node = db.get_storage_node_by_id(node_id) + cluster = db.get_cluster_by_id(owner.cluster_id) + except KeyError as e: + logger.error("Unfence of %s on %s: %s", node_id, device_id, e) + return False + if dev.serving_node() == node_id: + logger.error("Unfence of %s on %s refused: it is the serving node", node_id, device_id) + return False + try: + if not array.plugin_for_cluster(cluster).unfence(dev.lun_wwn, multi_attach.array_host(node)): + logger.error("Unfence of %s on LUN %s not acknowledged", node_id, dev.lun_wwn) + return False + except Exception as e: # noqa: BLE001 + logger.error("Unfence of %s on LUN %s failed: %s", node_id, dev.lun_wwn, e) + return False + + def _mut(n): + d = _device_on_owner(n, device_id) + if d is None or node_id not in d.fenced_node_ids: + return False + d.fenced_node_ids = [x for x in d.fenced_node_ids if x != node_id] + return True + + db.atomic_update(owner, _mut) + _event(owner.cluster_id, dev, f"Node {node_id} unfenced from LUN {dev.lun_wwn} of device {device_id}") + return True diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index 833b849ada..0fdb85bec4 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -1817,6 +1817,50 @@ def claim_cntlid_block(self, node, max_blocks): _NoTxnStore(kv), node_key, alloc_key, node.get_id(), live_node_ids, max_blocks, index_list) + # ---- Multi-attach LUN takeover lease (Single FDB Transaction) ---- + + LUN_TAKEOVER_LOCK_PREFIX = "lun_takeover_lock" + + @staticmethod + def _acquire_lun_takeover_lock_tx(tr, key, owner, now, ttl): + raw = tr.get(key).wait() + if raw.present(): + held = json.loads(bytes(raw).decode()) + if (held.get("owner") and held.get("owner") != owner + and now - float(held.get("at", 0)) <= ttl): + return False, held.get("owner") + tr.set(key, json.dumps({"owner": owner, "at": now}).encode()) + return True, None + + @staticmethod + def _release_lun_takeover_lock_tx(tr, key, owner): + raw = tr.get(key).wait() + if raw.present() and json.loads(bytes(raw).decode()).get("owner") == owner: + tr.clear(key) + + def acquire_lun_takeover_lock(self, device_id, owner, ttl): + """Take the takeover lease of one multi-attach LUN for ``owner``, so + two takeovers of the same LUN never run together, in this process or + in another. A lease older than ``ttl`` seconds (its holder died) is + taken over. Returns ``(True, None)`` or ``(False, holder)``.""" + key = f"{self.LUN_TAKEOVER_LOCK_PREFIX}/{device_id}".encode() + now = time.time() + kv = self.kv_store + if kv is not None and hasattr(kv, 'create_transaction'): + transactional = fdb.transactional(DBController._acquire_lun_takeover_lock_tx) + return transactional(kv, key, owner, now, ttl) + return DBController._acquire_lun_takeover_lock_tx(_NoTxnStore(kv), key, owner, now, ttl) + + def release_lun_takeover_lock(self, device_id, owner): + """Release the lease only if ``owner`` still holds it.""" + key = f"{self.LUN_TAKEOVER_LOCK_PREFIX}/{device_id}".encode() + kv = self.kv_store + if kv is not None and hasattr(kv, 'create_transaction'): + transactional = fdb.transactional(DBController._release_lun_takeover_lock_tx) + transactional(kv, key, owner) + return + DBController._release_lun_takeover_lock_tx(_NoTxnStore(kv), key, owner) + # ---- Generic atomic read-modify-write (Single FDB Transaction) ---- def _atomic_update_tx(self, tr, key, model_cls, mutate_fn, index_list): diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 835ceb360c..2499439572 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -195,6 +195,15 @@ def is_topology_owned(self) -> bool: # Requires raid_devices. Gates every multi-attach path (device NQN, # controller-ID ranges, hold timeouts, owner loopback). multi_attach: bool = False + # Multi-attach clusters: the storage array the LUNs live on. A LUN is + # taken over by another node only after its serving node is fenced at the + # array, through the plugin named here (entry point in the group + # simplyblock.array_plugins, see simplyblock_core.array). Empty = no + # plugin: a takeover is refused (never reopen a LUN unfenced). Credentials + # are resolved from the KMS in a later phase. + array_plugin: str = "" + array_endpoint: str = "" + array_options: dict = default_factory(dict) tls: bool = False tls_config: dict = default_factory(dict) is_re_balancing: bool = False diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index adc4cbddbd..2302187b65 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -127,6 +127,10 @@ class NVMeDevice(BaseModel): #: node_id -> {"path": by-id path of the LUN on that node, "bdev": the #: base bdev name there}. Only the owner's entry is filled so far. node_paths: dict = default_factory(dict) + #: Nodes a LUN takeover fenced from this LUN at the array. Phase 3 never + #: unfences on its own: a fenced node's host may still hold queued + #: writes. Such a node is no takeover candidate for this LUN. + fenced_node_ids: list = default_factory(list) def serving_node(self) -> str: """The node whose alceml serves this device now (the owner unless a diff --git a/simplyblock_core/multi_attach.py b/simplyblock_core/multi_attach.py index 0615d32c9c..36955b2bec 100644 --- a/simplyblock_core/multi_attach.py +++ b/simplyblock_core/multi_attach.py @@ -164,3 +164,138 @@ def own_multi_attach_devices(node, statuses) -> list: controller instead of the local alceml.""" return [d for d in (getattr(node, "nvme_devices", None) or []) if is_multi_attach(d) and d.status in statuses and d.alceml_bdev] + + +# ------------------------------------------------------------------ phase 3 +# LUN takeover (controllers/lun_takeover) and the paths it touches: a device +# may now be served by a node other than its owner (serving_node_id). + +_NODE_ONLINE = "online" + + +def served_elsewhere(device, node_id) -> bool: + """A multi-attach device served by a node other than ``node_id``: that + node must not build the device's alceml, subsystem or anything above the + base bdev, nor report the device lost when it goes away.""" + return is_multi_attach(device) and device.serving_node() != node_id + + +def fenced_from(device, node_id) -> bool: + """A takeover fenced ``node_id`` from the LUN of ``device`` at the array + and nothing unfenced it since.""" + return is_multi_attach(device) and node_id in (getattr(device, "fenced_node_ids", None) or []) + + +def not_servable_here(device, node_id) -> bool: + """``node_id`` must not build a stack for ``device``: another node serves + its LUN, or ``node_id`` is fenced from it (the takeover after its failure + did not complete; an operator unfences it -- lun_takeover.unfence_node -- + once its host is known to hold no queued writes).""" + return served_elsewhere(device, node_id) or fenced_from(device, node_id) + + +def array_host(node) -> str: + """A storage node's identity on the storage array: its hostname, without + the ``_`` suffix a containerised node's hostname carries.""" + host = getattr(node, "hostname", "") or "" + suffix = f"_{getattr(node, 'rpc_port', '')}" + if suffix != "_" and host.endswith(suffix): + host = host[: -len(suffix)] + return host + + +def standby_candidates(owner, failed_node_id) -> list[str]: + """Node ids that may take over ``owner``'s LUNs, in preference order: the + owner's LVS secondary, its tertiary, then the owner itself (when it is + alive while the node serving its LUN fails). The failed node is never a + candidate.""" + out: list[str] = [] + for nid in (getattr(owner, "secondary_node_id", ""), getattr(owner, "tertiary_node_id", ""), + owner.get_id()): + if nid and nid != failed_node_id and nid not in out: + out.append(nid) + return out + + +def devices_served_by(nodes, node_id) -> list: + """``(owner, device)`` for every multi-attach device of ``nodes`` whose + serving node is ``node_id`` (its own LUNs it still serves, and LUNs it took + over).""" + return [(owner, dev) for owner in nodes for dev in (owner.nvme_devices or []) + if is_multi_attach(dev) and dev.serving_node() == node_id] + + +def served_by_online_node(owner, nodes_by_id) -> list: + """``owner``'s multi-attach devices served by another node that is + ONLINE: reachable although the owner itself is down.""" + out = [] + for dev in getattr(owner, "nvme_devices", None) or []: + if not served_elsewhere(dev, owner.get_id()): + continue + server = nodes_by_id.get(dev.serving_node()) + if server is not None and server.status == _NODE_ONLINE: + out.append(dev) + return out + + +def normalize_wwn(value) -> str: + v = (value or "").strip().lower() + return v.removeprefix("0x") + + +def lun_keys(device) -> set: + """Identities a host may report for the LUN of ``device``: its WWN and + its serial (lsblk falls back to the WWN for the serial).""" + return {k for k in (normalize_wwn(getattr(device, "lun_wwn", "")), + normalize_wwn(getattr(device, "serial_number", ""))) if k} + + +def foreign_lun_keys(nodes, node_id) -> set: + """LUN identities of the multi-attach devices of every node but + ``node_id``. A device with one of them, found on ``node_id``'s host, is + another node's LUN (all LUNs are presented to every node): it is never a + new device of ``node_id`` and is never formatted.""" + keys: set = set() + for node in nodes: + if node.get_id() == node_id: + continue + for dev in node.nvme_devices or []: + if is_multi_attach(dev): + keys |= lun_keys(dev) + return keys + + +def is_foreign_lun(device, foreign_keys) -> bool: + return bool(foreign_keys) and bool(lun_keys(device) & foreign_keys) + + +def pick_lun_path(entries, device): + """The path to open the LUN of ``device`` with on a takeover node, from + the node agent's ``lun_info`` answer. Returns ``(path, "")`` or + ``(None, reason)``. + + Only disks whose block 0 carries the alceml signature (read just now) and + that are at least as large as the device qualify. The by-id path the owner + opened it with is preferred when the node has it too (a WWN-derived by-id + name is the same on every host); otherwise exactly one qualifying disk is + required -- several host paths to one LUN are not chosen between.""" + if not entries: + return None, "the LUN is not visible on this node" + readable = [e for e in entries if e.get("has_alceml_signature") is True] + if not readable: + states = sorted({str(e.get("has_alceml_signature")) for e in entries}) + return None, f"no path to the LUN carries the alceml signature (signature: {', '.join(states)})" + size = int(getattr(device, "size", 0) or 0) + sized = [e for e in readable if int(e.get("size") or 0) >= size] + if not sized: + return None, f"the LUN is smaller here than the device ({size} bytes)" + owner_path = ((getattr(device, "node_paths", None) or {}).get(device.node_id, {}).get("path") + or getattr(device, "by_id_path", "") or "") + if owner_path.startswith("/dev/disk/by-id/"): + for e in sized: + if owner_path in (e.get("by_id_paths") or []) or owner_path == e.get("by_id_path"): + return owner_path, "" + if len(sized) == 1: + e = sized[0] + return e.get("by_id_path") or e.get("device_path"), "" + return None, f"{len(sized)} disks carry the LUN and none has the owner's path {owner_path!r}" diff --git a/simplyblock_core/services/health_check_service.py b/simplyblock_core/services/health_check_service.py index 0f33f7012a..50a9e46ab5 100644 --- a/simplyblock_core/services/health_check_service.py +++ b/simplyblock_core/services/health_check_service.py @@ -3,7 +3,13 @@ import time from datetime import datetime -from simplyblock_core import constants, db_controller, storage_node_ops, utils +from simplyblock_core import ( + constants, + db_controller, + multi_attach, + storage_node_ops, + utils, +) from simplyblock_core.controllers import ( device_controller, device_events, @@ -291,6 +297,11 @@ def _group_devices(snode=snode): if device.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE]: logger.info(f"Device skipped: {device.get_id()} status: {device.status}") continue + if multi_attach.served_elsewhere(device, snode.get_id()): + # Multi-attach: another node took the LUN over; this node + # has no stack for it (its loopback is checked below). + logger.info(f"Device skipped: {device.get_id()} served by {device.serving_node()}") + continue passed = True @@ -349,7 +360,11 @@ def _group_devices(snode=snode): for remote_device in snode.remote_devices: org_dev = db.get_storage_device_by_id(remote_device.get_id()) - org_node = db.get_storage_node_by_id(remote_device.node_id) + # The node exporting the device: its owner, or the node a + # multi-attach takeover moved its LUN to. + org_node = db.get_storage_node_by_id( + org_dev.serving_node() if multi_attach.is_multi_attach(org_dev) + else remote_device.node_id) # Only treat a missing remote device as a fault when the owning node # is ONLINE/DOWN/UNREACHABLE. If the owner is mid-transition (restart, # shutdown, ...) the connection is expected to be gone — skip it. @@ -360,7 +375,9 @@ def _group_devices(snode=snode): # connect. _peer_connections_relevant also admits # UNREACHABLE, which is fine for judging whether a missing # connection is a fault and wrong for deciding to dial out. - if org_dev.nvmf_multipath: + # Multi-attach: also single-path, the controller may + # still miss the path to the node that took it over. + if org_dev.nvmf_multipath or multi_attach.is_multi_attach(org_dev): ctrl_name = f"remote_{org_dev.alceml_bdev}" if org_dev.alceml_bdev else None if ctrl_name and health_controller.repairs_allowed(org_node): device_repair_jobs.append((ctrl_name, org_dev, snode)) diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index fc039e385d..556385e2e2 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -10,6 +10,7 @@ constants, db_controller, floors, + multi_attach, storage_network, storage_node_ops, utils, @@ -1164,10 +1165,7 @@ def _force_offline_lost_site_node(node): tasks_controller.cancel_pending_node_restart_tasks(node.cluster_id, node.get_id()) storage_node_ops.set_node_status(node.get_id(), StorageNode.STATUS_OFFLINE, caused_by="sync_site_lost") - for dev in node.nvme_devices: - if dev.status in (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, - NVMeDevice.STATUS_CANNOT_ALLOCATE): - device_controller.device_set_unavailable(dev.get_id()) + _set_node_devices_unavailable(node, "site lost") def _drain_nodes(nodes, what, gone=None) -> bool: @@ -1830,6 +1828,86 @@ def _take_over_leaderless_lvstores(snode): lvs_name, leader.get_id(), snode.get_id()) +#: Device states that are lost with their node (set unavailable). +_DEV_SERVING_STATES = (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE) + +#: device id -> running LUN takeover thread (multi-attach clusters). +lun_takeover_threads: dict[str, threading.Thread] = {} + + +def _lun_takeover_worker(device_id, failed_node_id, reason, started): + """Take over one LUN; mark the device unavailable when it was not taken + over (the regular node-loss behaviour). When the LUN was fenced but no + standby could read it, the peers keep holding I/O: the device is marked + unavailable only when their hold has run out.""" + from simplyblock_core.controllers import lun_takeover + try: + outcome = lun_takeover.take_over_lun_outcome(device_id, failed_node_id, reason) + except Exception as e: # noqa: BLE001 + logger.error("LUN takeover of %s raised: %s", device_id, e) + outcome = lun_takeover.FAILED + if outcome in (lun_takeover.TAKEN_OVER, lun_takeover.IN_PROGRESS): + return + if outcome == lun_takeover.NO_STANDBY: + wait = started + constants.MULTI_ATTACH_HOLD_SEC - time.monotonic() + if wait > 0: + time.sleep(wait) + try: + dev = db.get_storage_device_by_id(device_id) + except KeyError: + return + if dev.serving_node() != failed_node_id or dev.status not in _DEV_SERVING_STATES: + return + logger.warning("LUN of device %s not taken over (%s); setting it unavailable", + device_id, outcome) + device_controller.device_set_unavailable(device_id) + + +def _start_lun_takeover(device_id, failed_node_id, reason): + existing = lun_takeover_threads.get(device_id) + if existing is not None and existing.is_alive(): + logger.info("LUN takeover of %s already running", device_id) + return + t = threading.Thread(target=_lun_takeover_worker, + args=(device_id, failed_node_id, reason, time.monotonic()), + name=f"lun-takeover-{device_id[:8]}", daemon=True) + lun_takeover_threads[device_id] = t + t.start() + + +def _set_node_devices_unavailable(node, reason): + """The devices lost with ``node``: set unavailable, as always. + + Multi-attach clusters (only): a LUN the node served -- its own or one it + had taken over -- is taken over by a standby instead, each in its own + thread, started before anything else so the fence comes first and the + takeover fits in the peers' hold; a LUN of the node that another node + serves is not lost with it and is left alone. Only devices whose takeover + fails are set unavailable (by the takeover thread).""" + try: + cluster = db.get_cluster_by_id(node.cluster_id) + except Exception as e: # noqa: BLE001 + # Unknown cluster mode: the regular behaviour (never a takeover). + logger.warning("Cluster of %s not read (%s); devices set unavailable", node.get_id(), e) + cluster = None + if not multi_attach.is_multi_attach(cluster): + for dev in node.nvme_devices: + if dev.status in _DEV_SERVING_STATES: + # Default cause is correct: node going offline cascades + # to all its devices, that does not count as a flap. + device_controller.device_set_unavailable(dev.get_id()) + return + node_id = node.get_id() + for _owner, dev in multi_attach.devices_served_by( + db.get_storage_nodes_by_cluster_id(node.cluster_id), node_id): + if dev.status in _DEV_SERVING_STATES: + _start_lun_takeover(dev.get_id(), node_id, reason) + for dev in node.nvme_devices: + if dev.status in _DEV_SERVING_STATES and not multi_attach.is_multi_attach(dev): + device_controller.device_set_unavailable(dev.get_id()) + + def _ana_failover_rearm(node_id) -> None: """Forget the one-shot marker so the next offline episode re-triggers.""" _ana_failover_applied.discard(node_id) @@ -1855,12 +1933,9 @@ def set_node_offline(node): StorageNode.STATUS_RESTARTING]: try: storage_node_ops.set_node_status(node.get_id(), StorageNode.STATUS_OFFLINE) - for dev in node.nvme_devices: - if dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, - NVMeDevice.STATUS_CANNOT_ALLOCATE]: - # Default cause is correct: node going offline cascades - # to all its devices, that does not count as a flap. - device_controller.device_set_unavailable(dev.get_id()) + # Multi-attach: LUN takeovers start here, before the LVS ANA + # failover below (they run in parallel with it). + _set_node_devices_unavailable(node, "node offline") except Exception as e: logger.debug("Setting node to OFFLINE state failed") logger.error(e) @@ -2162,10 +2237,7 @@ def set_node_schedulable(node): # initiate shutdown # initiate restart # tasks_controller.add_node_to_auto_restart(node) - for dev in node.nvme_devices: - if dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, - NVMeDevice.STATUS_CANNOT_ALLOCATE]: - device_controller.device_set_unavailable(dev.get_id()) + _set_node_devices_unavailable(node, "node schedulable") update_cluster_status(node.cluster_id) except Exception as e: logger.debug("Setting node to SCHEDULABLE state failed") diff --git a/simplyblock_core/snode_client.py b/simplyblock_core/snode_client.py index 99ee2c5851..d4aa934293 100644 --- a/simplyblock_core/snode_client.py +++ b/simplyblock_core/snode_client.py @@ -252,6 +252,12 @@ def wipe_block_device(self, device_name): return self._request("POST", "wipe_block_device", {"device_name": device_name}) + def lun_info(self, wwn): + """Multi-attach: the local disks carrying LUN ``wwn``, each with + ``has_alceml_signature`` (block 0 read, nothing written). + Returns ``(result, error)``.""" + return self._request("POST", "lun_info", {"wwn": wwn}) + def rescan_block_device(self, device_path): """Re-read the size of a grown whole-disk block device (lblk). diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 0d767a6a54..cc0b21f1d1 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -1089,6 +1089,16 @@ def _collect_attached_ips(ctrlr_list): return attached +def _serving_node_id(device) -> str: + """The node exporting ``device``: ``serving_node()`` for an NVMeDevice + (the owner unless a multi-attach takeover moved its LUN), ``node_id`` for + any other device record (JM devices).""" + serving = getattr(device, "serving_node", None) + if callable(serving) and multi_attach.is_multi_attach(device): + return serving() + return device.node_id + + def connect_device(name: str, device: NVMeDevice, node: StorageNode, attach_timeout: float | None = None): """Connect snode to device @@ -1108,6 +1118,11 @@ def connect_device(name: str, device: NVMeDevice, node: StorageNode, attach_time expected_ips = [ip.strip() for ip in (device.nvmf_ip or "").split(",") if ip.strip()] is_multipath = bool(device.nvmf_multipath) and len(expected_ips) >= 2 + # Multi-attach: the controller can hold a dead path of the node that + # served the LUN before a takeover while its bdev lives on, so the + # controller is always inspected for the serving node's paths. + if multi_attach.is_multi_attach(device): + is_multipath = True rpc_client = node.rpc_client() # Fast path: bdev already present. Only safe for single-path devices — for @@ -1171,7 +1186,14 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r # the budget, raise so the calling task suspends and retries. _TRANSIENT_STATES = ("failed", "resetting", "deleting", "reconnect_is_delayed") states: list[str] = [] + # Multi-attach: a path to the node that served the LUN before a + # takeover stays "resetting"/"reconnect_is_delayed" for the whole + # hold (MULTI_ATTACH_HOLD_SEC); waiting for it to settle would keep + # the new path from being added, which is what ends the hold. for _attempt in range(5): + if multi_attach.is_multi_attach(device): + logger.info(f"Controller found: {name}, multi-attach, not waiting on path states") + break if not ret: # The module destructed the controller on its own; the # fresh-attach path below takes over. @@ -1189,7 +1211,9 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r raise RuntimeError(f"Controller: {name}, status is {states}") db_ctrl = DBController() - target_node = db_ctrl.get_storage_node_by_id(device.node_id) + # The node exporting the device: its owner, or the node a multi-attach + # takeover moved its LUN to. + target_node = db_ctrl.get_storage_node_by_id(_serving_node_id(device)) if target_node is not None and target_node.active_rdma: tr_type = "RDMA" elif target_node is not None and target_node.active_tcp: @@ -1238,7 +1262,12 @@ def _connect_device_attach(name, device, node: StorageNode, rpc_client, attach_r dial_backoff.record_failure(ip) logger.warning(f"Failed to attach controller {name} via {ip}: {e}") - if device.nvmf_multipath and bdev_name: + if multi_attach.is_multi_attach(device) and bdev_name: + # One node serves the LUN at a time; its paths are the only + # valid ones (a takeover adds another node's, then drops the + # old). + rpc_client.bdev_nvme_set_multipath_policy(bdev_name, "active_passive") + elif device.nvmf_multipath and bdev_name: rpc_client.bdev_nvme_set_multipath_policy(bdev_name, "active_active") if not bdev_name: @@ -1326,7 +1355,10 @@ def repair_multipath_controller(name: str, device, node: StorageNode): (i.e. the source NVMeDevice / JMDevice on the target node — NOT a ``RemoteJMDevice``, which strips those fields). """ - if not getattr(device, 'nvmf_multipath', False): + # Multi-attach: a single-path controller too can lack the path to the + # node a takeover moved the LUN to. + ma = multi_attach.is_multi_attach(device) + if not getattr(device, 'nvmf_multipath', False) and not ma: return True nvmf_ip = getattr(device, 'nvmf_ip', None) @@ -1340,7 +1372,7 @@ def repair_multipath_controller(name: str, device, node: StorageNode): return False expected_ips = {ip.strip() for ip in nvmf_ip.split(",") if ip.strip()} - if len(expected_ips) < 2: + if len(expected_ips) < 2 and not ma: return True # not actually multipath rpc_client = node.rpc_client() @@ -1349,7 +1381,7 @@ def repair_multipath_controller(name: str, device, node: StorageNode): return True # controller gone, connect_device will handle full reconnect db_ctrl = DBController() - target_node = db_ctrl.get_storage_node_by_id(device.node_id) + target_node = db_ctrl.get_storage_node_by_id(_serving_node_id(device)) if target_node is None: return False if target_node.active_rdma: @@ -2301,6 +2333,19 @@ def _prepare_cluster_devices_on_restart(snode: StorageNode, clear_data=False): logger.debug(f"Device is skipped: {nvme.get_id()}, status: {nvme.status}") continue + if multi_attach.not_servable_here(nvme, snode.get_id()): + # Multi-attach: another node took the LUN over and serves it. This + # node keeps the record and reaches the LUN through its loopback + # controller to the serving node; opening a second alceml on the + # LUN here would make two writers. No failback in this phase. + # Or this node is still fenced from the LUN (a takeover after its + # failure did not complete): the LUN cannot be opened until an + # operator unfences it. + logger.info(f"Device {nvme.get_id()} is not served by this node (serving node " + f"{nvme.serving_node()}, fenced: {multi_attach.fenced_from(nvme, snode.get_id())}); " + f"not building its stack here") + continue + jobs.append((nvme, _create_storage_device_stack, (rpc_client, nvme, snode, not clear_data,))) # Every device that reaches this point was found on the node and is @@ -2517,17 +2562,26 @@ def _connect_to_remote_devs( # its distribs through a loopback controller (remote_alceml_n1) like # every peer's, so they are connected here too. Empty on other clusters. own_loopback = multi_attach.own_multi_attach_devices(this_node, allowed_dev_statuses) + # Multi-attach clusters: a down owner's LUNs that another node took over + # are reachable through that node. Empty on other clusters. + nodes_by_id = {n.get_id(): n for n in nodes} + taken_over = {n.get_id(): multi_attach.served_by_online_node(n, nodes_by_id) + for n in nodes if n.status not in allowed_node_statuses} # connect to remote devs for node_index, node in enumerate(nodes): if node.get_id() == this_node.get_id(): if not own_loopback: continue + node_devices = own_loopback elif node.status not in allowed_node_statuses: - continue + node_devices = taken_over.get(node.get_id()) + if not node_devices: + continue + else: + node_devices = node.nvme_devices if only_node_id and node.get_id() != only_node_id: continue logger.info(f"Connecting to node {node.get_id()}") - node_devices = own_loopback if node.get_id() == this_node.get_id() else node.nvme_devices for index, dev in enumerate(node_devices): if dev.status not in allowed_dev_statuses: @@ -2637,8 +2691,13 @@ def _find_remote_bdev(dev): if node.get_id() == this_node.get_id(): if not own_loopback: continue + node_devices = own_loopback elif node.status not in allowed_node_statuses: - continue + node_devices = taken_over.get(node.get_id()) + if not node_devices: + continue + else: + node_devices = node.nvme_devices if only_node_id and node.get_id() != only_node_id: # DELTA mode: records for every other node were carried over # untouched above — probing their bdevs anyway made each delta @@ -2647,7 +2706,7 @@ def _find_remote_bdev(dev): # ~25 min, 2026-07-13 validation run). Verify only the delta # owner's devices. continue - for dev in (own_loopback if node.get_id() == this_node.get_id() else node.nvme_devices): + for dev in node_devices: if dev.get_id() in remote_device_ids: continue if dev.status not in allowed_dev_statuses: @@ -2864,7 +2923,9 @@ def sync_remote_devices_from_spdk(this_node: StorageNode): # device (O(cluster devices) per call). None -> per-device fallback. _sweep_bdev_names = _fetch_bdev_name_set(rpc_client) - for peer in db_controller.get_storage_nodes_by_cluster_id(fresh_node.cluster_id): + peers = db_controller.get_storage_nodes_by_cluster_id(fresh_node.cluster_id) + peers_by_id = {p.get_id(): p for p in peers} + for peer in peers: if peer.get_id() == fresh_node.get_id(): # Multi-attach clusters: own array LUNs (loopback controllers). peer_devices = multi_attach.own_multi_attach_devices( @@ -2873,7 +2934,10 @@ def sync_remote_devices_from_spdk(this_node: StorageNode): if not peer_devices: continue elif peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN, StorageNode.STATUS_RESTARTING]: - continue + # Multi-attach: its LUNs another node took over (empty otherwise). + peer_devices = multi_attach.served_by_online_node(peer, peers_by_id) + if not peer_devices: + continue else: peer_devices = peer.nvme_devices for dev in peer_devices: @@ -2951,7 +3015,9 @@ def reconnect_dropped_remote_devs(this_node: StorageNode): changed = False all_ok = True - for peer in db_controller.get_storage_nodes_by_cluster_id(fresh_node.cluster_id): + peers = db_controller.get_storage_nodes_by_cluster_id(fresh_node.cluster_id) + peers_by_id = {p.get_id(): p for p in peers} + for peer in peers: if peer.get_id() == fresh_node.get_id(): # Multi-attach clusters: own array LUNs (loopback controllers). peer_devices = multi_attach.own_multi_attach_devices( @@ -2961,7 +3027,10 @@ def reconnect_dropped_remote_devs(this_node: StorageNode): continue elif peer.status not in [StorageNode.STATUS_ONLINE, StorageNode.STATUS_DOWN, StorageNode.STATUS_UNREACHABLE]: - continue + # Multi-attach: its LUNs another node took over (empty otherwise). + peer_devices = multi_attach.served_by_online_node(peer, peers_by_id) + if not peer_devices: + continue else: peer_devices = peer.nvme_devices for dev in peer_devices: @@ -5029,6 +5098,9 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, nvme_devs = utils.addAioDevices(rpc_client, snode, lblk_resolved) # Multi-attach clusters only: flag the array LUNs (no-op otherwise). multi_attach.mark_lun_devices(cluster, snode, nvme_devs, lblk_resolved) + if multi_attach.is_multi_attach(cluster): + # Another node's LUN is never a device of this node. + nvme_devs = drop_foreign_luns(snode, nvme_devs) else: if not snode.ssd_pcie: node_info, _ = snode_api.info() @@ -9084,6 +9156,37 @@ def _new_restart_claim_token(): return f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}" +def _foreign_lun_keys(snode): + """LUN identities of the other nodes' multi-attach devices, on a + multi-attach cluster; an empty set otherwise (no DB read).""" + db_controller = DBController() + try: + cluster = db_controller.get_cluster_by_id(snode.cluster_id) + except KeyError: + return set() + if not multi_attach.is_multi_attach(cluster): + return set() + return multi_attach.foreign_lun_keys( + db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id), snode.get_id()) + + +def drop_foreign_luns(snode, nvme_devs): + """``nvme_devs`` without the multi-attach LUNs of other nodes (add-node + discovery: they must never become devices of ``snode``, which would + format them). Unchanged on any other cluster.""" + foreign = _foreign_lun_keys(snode) + if not foreign: + return nvme_devs + kept = [] + for dev in nvme_devs: + if multi_attach.is_foreign_lun(dev, foreign): + logger.warning(f"Block device {dev.device_path or dev.device_name} (serial " + f"{dev.serial_number}) is another node's multi-attach LUN; skipped") + continue + kept.append(dev) + return kept + + def _reconcile_discovered_devices(snode, nvme_devs): """Match the devices just discovered on the host (`nvme_devs`, from addNvmeDevices/addAioDevices) against this node's known NVMeDevice @@ -9108,6 +9211,10 @@ def _reconcile_discovered_devices(snode, nvme_devs): known_devices_sn = [] admin_removed_sn = set() devices_sn_dict = {d.serial_number: d for d in nvme_devs} + # Multi-attach clusters: every LUN is presented to every node. A LUN of + # another node's device is never a new device here (it would be + # formatted). Empty on any other cluster. + foreign_luns = _foreign_lun_keys(snode) for db_dev in snode.nvme_devices: if db_dev.admin_removed: admin_removed_sn.add(db_dev.serial_number) @@ -9137,6 +9244,13 @@ def _reconcile_discovered_devices(snode, nvme_devs): db_dev.md_supported = found_dev.md_supported active_devices.append(db_dev) + elif multi_attach.not_servable_here(db_dev, snode.get_id()): + # Multi-attach: the LUN is served by the node that took it over, + # or this node is fenced from it at the array; not seeing it does + # not mean it is gone. The record stays as it is. + logger.info(f"Device not found: {db_dev.get_id()}, served by node " + f"{db_dev.serving_node()}; record kept") + active_devices.append(db_dev) else: logger.info(f"Device not found: {db_dev.get_id()}") if db_dev.status == NVMeDevice.STATUS_NEW: @@ -9155,6 +9269,11 @@ def _reconcile_discovered_devices(snode, nvme_devs): logger.info(f"JM device found: {snode.jm_device.get_id()}") snode.jm_device.nvme_bdev = dev.nvme_bdev + elif dev.serial_number not in known_devices_sn and multi_attach.is_foreign_lun(dev, foreign_luns): + logger.warning(f"Block device {dev.device_path or dev.device_name} (serial " + f"{dev.serial_number}) is another node's multi-attach LUN; " + f"not added (never formatted)") + elif dev.serial_number not in known_devices_sn: if dev.serial_number in admin_removed_sn: logger.warning( @@ -9869,6 +9988,10 @@ def _jm_reconcile(): snode = db_controller.get_storage_node_by_id(snode.get_id()) for db_dev in snode.nvme_devices: + if multi_attach.not_servable_here(db_dev, snode.get_id()): + # Multi-attach: its state follows the node serving it, not this + # restart (this node built nothing for it). + continue if db_dev.status in [NVMeDevice.STATUS_UNAVAILABLE, NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_CANNOT_ALLOCATE, NVMeDevice.STATUS_READONLY]: db_dev.status = NVMeDevice.STATUS_ONLINE @@ -10578,6 +10701,10 @@ def _detach_remote_controllers_from_peers(snode: StorageNode, db_controller): """ shutting_down_id = snode.get_id() all_peers = db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id) + # Multi-attach: controllers to this node's LUNs that another node serves + # lead to that node, not to this one; they stay. + served_elsewhere_ids = {d.get_id() for d in snode.nvme_devices + if multi_attach.served_elsewhere(d, shutting_down_id)} peers = [ p for p in all_peers if p.get_id() != shutting_down_id @@ -10603,6 +10730,8 @@ def _detach_one_peer(peer): for rem_dev in (peer.remote_devices or []): if rem_dev.node_id != shutting_down_id: continue + if rem_dev.get_id() in served_elsewhere_ids: + continue bdev_name = rem_dev.remote_bdev or "" if bdev_name.endswith("n1"): ctrl_names.append(bdev_name[:-2]) @@ -10813,6 +10942,10 @@ def _broadcast_devices_unavailable(snode): NVMeDevice.STATUS_READONLY, ]: continue + if multi_attach.served_elsewhere(dev, snode.get_id()): + # Multi-attach: another node serves this LUN; it is not lost + # with this node. + continue try: # Default cause (CAUSE_OTHER): a node-driven shutdown # must not count against the per-device flap budget. diff --git a/simplyblock_web/api/internal/storage_node/docker.py b/simplyblock_web/api/internal/storage_node/docker.py index d540818087..831ba5a9d9 100644 --- a/simplyblock_web/api/internal/storage_node/docker.py +++ b/simplyblock_web/api/internal/storage_node/docker.py @@ -652,6 +652,22 @@ def wipe_block_device(body: _WipeBlockDeviceParams): return utils.get_response(True) +class _LunInfoParams(BaseModel): + wwn: str + + +@api.post('/lun_info', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'array', + 'items': {'type': 'object', 'additionalProperties': True}, + })}}}, +}) +def lun_info(body: _LunInfoParams): + """Multi-attach array LUN mode: the local disks carrying LUN ``wwn``, with + whether block 0 holds the alceml signature. Read-only.""" + return utils.get_response(node_utils.find_lun_devices(body.wwn)) + + class _RescanBlockDeviceParams(BaseModel): device_path: str diff --git a/simplyblock_web/api/internal/storage_node/kubernetes.py b/simplyblock_web/api/internal/storage_node/kubernetes.py index c2af4828a2..394d8fdf75 100644 --- a/simplyblock_web/api/internal/storage_node/kubernetes.py +++ b/simplyblock_web/api/internal/storage_node/kubernetes.py @@ -350,6 +350,22 @@ def wipe_block_device(body: _WipeBlockDeviceParams): return utils.get_response(True) +class _LunInfoParams(BaseModel): + wwn: str + + +@api.post('/lun_info', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'array', + 'items': {'type': 'object', 'additionalProperties': True}, + })}}}, +}) +def lun_info(body: _LunInfoParams): + """Multi-attach array LUN mode: the local disks carrying LUN ``wwn``, with + whether block 0 holds the alceml signature. Read-only.""" + return utils.get_response(node_utils.find_lun_devices(body.wwn)) + + class _RescanBlockDeviceParams(BaseModel): device_path: str diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index 1abefb2559..7125d18dbb 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -438,6 +438,43 @@ def simplyblock_data_signature(device_name: str) -> bool | None: return head.startswith(ALCEML_STORAGE_SIGNATURE) +def _normalize_wwn(value: str) -> str: + v = (value or "").strip().lower() + return v.removeprefix("0x") + + +def find_lun_devices(wwn: str) -> list[dict]: + """The whole disks of this host that carry LUN ``wwn`` (multi-attach array + LUN mode): every lsblk disk whose WWN, or serial (the inventory falls back + to the WWN for it), matches, case-insensitive and with or without ``0x``. + + One dict per match: ``name``, ``device_path``, ``by_id_path``, + ``by_id_paths``, ``size`` (bytes), ``ro`` and ``has_alceml_signature`` -- + True when block 0 carries the alceml storage signature, False when it was + read and does not, None when it could not be read (a fenced or failed + path). Read-only: nothing is written to the device. Several matches mean + several host paths to the LUN; the caller chooses.""" + want = _normalize_wwn(wwn) + if not want: + return [] + found = [] + for dev in get_block_devices_info(): + if dev.get("type") != "disk": + continue + if want not in (_normalize_wwn(dev.get("wwn", "")), _normalize_wwn(dev.get("serial", ""))): + continue + found.append({ + "name": dev["name"], + "device_path": dev["device_path"], + "by_id_path": dev.get("by_id_path", ""), + "by_id_paths": dev.get("by_id_paths", []), + "size": dev.get("size", 0), + "ro": dev.get("ro", False), + "has_alceml_signature": simplyblock_data_signature(dev["name"]), + }) + return found + + def _logical_block_size(disk_name: str) -> int: """Logical sector size of a disk, in bytes: the unit sgdisk counts in. sysfs reports partition start and size in 512-byte units regardless.""" diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py new file mode 100644 index 0000000000..3241a187b8 --- /dev/null +++ b/tests/unit/test_lun_takeover.py @@ -0,0 +1,706 @@ +"""Multi-attach array LUN mode, phase 3: LUN takeover by a standby node. + + - array plugin interface: registry, entry points, the fake is never + registered by default. + - controllers/lun_takeover: fence first, standby choice, open-only alceml, + export under the same NQN/namespace, record, paths; fence failure, no + plugin, no readable standby, idempotent re-run, concurrent takeover. + - storage_node_monitor: takeovers on node loss (multi-attach only), + unavailable on failure, regular clusters unchanged. + - storage_node_ops: owner restart skips LUNs served elsewhere; discovery + never adopts (formats) another node's LUN; connect uses the serving node. + - device_controller: ONLINE while the owner is down only via a takeover. +""" + +import copy +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import array, constants, multi_attach, storage_node_ops +from simplyblock_core.array.fake import FakeArrayPlugin +from simplyblock_core.controllers import device_controller, lun_takeover +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.iface import IFace +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import storage_node_monitor + +DEV_ID = "1b4e28ba-2fa1-41d2-883f-0016d3cca427" +WWN = "0x600a098038314b6f2b5d4e6a4f2d6b31" +BY_ID = "/dev/disk/by-id/wwn-0x600a098038314b6f2b5d4e6a4f2d6b31" +SIZE = 8 * 1024 ** 3 +PLUGIN = "fake-test" + + +# ------------------------------------------------------------------ fakes + +class FakeSpdk: + """The RPC surface a takeover uses, with bdev/subsystem state.""" + + def __init__(self, name): + self.name = name + self.bdevs = {} + self.subsystems = {} + self.calls = [] + self.fail = set() + + def _rec(self, op, *args, **kwargs): + self.calls.append((op, args, kwargs)) + return op not in self.fail + + def ops(self, op): + return [c for c in self.calls if c[0] == op] + + def bdev_get(self, name): + return copy.deepcopy(self.bdevs.get(name)) + + def bdev_aio_create(self, name, filename, block_size=0): + if not self._rec("bdev_aio_create", name, filename, block_size): + return None + self.bdevs[name] = {"name": name, "block_size": 4096, "num_blocks": SIZE // 4096} + return name + + def bdev_alceml_create(self, name, base, uuid, **kwargs): + if not self._rec("bdev_alceml_create", name, base, uuid, **kwargs): + return None + self.bdevs[name] = {"name": name} + return name + + def bdev_PT_NoExcl_create(self, name, base): + self._rec("bdev_PT_NoExcl_create", name, base) + self.bdevs[name] = {"name": name, "aliases": [f"pt-uuid-{self.name}"]} + return True + + def subsystem_get(self, nqn): + return copy.deepcopy(self.subsystems.get(nqn)) + + def subsystem_create(self, nqn, sn, mn, **kwargs): + self._rec("subsystem_create", nqn, sn, mn, **kwargs) + self.subsystems[nqn] = {"nqn": nqn, "listen_addresses": [], "namespaces": []} + return True + + def listeners_create(self, nqn, trtype, traddr, port): + self._rec("listeners_create", nqn, trtype, traddr, port) + self.subsystems[nqn]["listen_addresses"].append({"traddr": traddr}) + return True + + def nvmf_subsystem_add_ns(self, nqn, pt, uuid=None): + if not self._rec("nvmf_subsystem_add_ns", nqn, pt, uuid): + return None + self.subsystems[nqn]["namespaces"].append({"name": pt, "uuid": uuid}) + return 1 + + def alceml_set_qos_weights(self, weights): + return True + + def bdev_nvme_attach_controller(self, *args, **kwargs): + return self._rec("attach", *args, **kwargs) + + def bdev_nvme_detach_controller(self, *args, **kwargs): + return self._rec("detach", *args, **kwargs) + + def bdev_nvme_set_multipath_policy(self, *args, **kwargs): + return self._rec("policy", *args, **kwargs) + + def subsystem_delete(self, nqn): + self._rec("subsystem_delete", nqn) + self.subsystems.pop(nqn, None) + + def bdev_PT_NoExcl_delete(self, name): + self._rec("bdev_PT_NoExcl_delete", name) + self.bdevs.pop(name, None) + + def bdev_alceml_delete(self, name): + self._rec("bdev_alceml_delete", name) + self.bdevs.pop(name, None) + + def bdev_aio_delete(self, name): + self._rec("bdev_aio_delete", name) + self.bdevs.pop(name, None) + + +class FakeDB: + def __init__(self, cluster, nodes): + self.cluster = cluster + self.nodes = {n.get_id(): n for n in nodes} + self.lock_holder = None + + def get_cluster_by_id(self, _cid): + return self.cluster + + def get_storage_nodes_by_cluster_id(self, _cid): + return list(self.nodes.values()) + + def get_storage_node_by_id(self, nid): + if nid not in self.nodes: + raise KeyError(nid) + return self.nodes[nid] + + def get_storage_device_by_id(self, did): + for n in self.nodes.values(): + for d in n.nvme_devices: + if d.get_id() == did: + return d + raise KeyError(did) + + def atomic_update(self, obj, fn): + fresh = self.nodes[obj.get_id()] + fn(fresh) + return fresh + + def acquire_lun_takeover_lock(self, _did, owner, _ttl): + if self.lock_holder and self.lock_holder != owner: + return False, self.lock_holder + self.lock_holder = owner + return True, None + + def release_lun_takeover_lock(self, _did, owner): + if self.lock_holder == owner: + self.lock_holder = None + + +def _cluster(plugin=PLUGIN, multi=True): + cl = Cluster() + cl.uuid = "cluster-1" + cl.nqn = f"{constants.CLUSTER_NQN}:cluster-1" + cl.distr_ndcs, cl.distr_npcs = 1, 0 + cl.multi_attach = multi + cl.array_plugin = plugin + cl.status = Cluster.STATUS_ACTIVE + return cl + + +def _node(nid, ip, block, status=StorageNode.STATUS_ONLINE): + n = StorageNode() + n.uuid = nid + n.cluster_id = "cluster-1" + n.hostname = f"host-{nid}_5260" + n.rpc_port = 5260 + n.status = status + n.nvmf_port = 4420 + n.cntlid_block = block + n.active_tcp = True + nic = IFace() + nic.ip4_address = ip + n.data_nics = [nic] + n.spdk = FakeSpdk(nid) + n.rpc_client = lambda *a, _s=n.spdk, **k: _s + n.agent = MagicMock() + n.client = lambda *a, _c=n.agent, **k: _c + return n + + +def _lun_entry(signature=True, path=BY_ID, size=SIZE): + return {"name": "sdc", "device_path": "/dev/sdc", "by_id_path": path, + "by_id_paths": [path], "size": size, "has_alceml_signature": signature} + + +def _bed(plugin=PLUGIN, multi=True, dev_status=NVMeDevice.STATUS_ONLINE): + """Owner A (serves its LUN), secondary B, tertiary C, peer D.""" + a = _node("node-a", "10.0.0.1", 0) + b = _node("node-b", "10.0.0.2", 1) + c = _node("node-c", "10.0.0.3", 2) + d = _node("node-d", "10.0.0.4", 3) + a.secondary_node_id, a.tertiary_node_id = "node-b", "node-c" + dev = NVMeDevice() + dev.uuid = DEV_ID + dev.node_id = "node-a" + dev.cluster_id = "cluster-1" + dev.status = dev_status + dev.size = SIZE + dev.bdev_type = "aio" + dev.serial_number = WWN + dev.multi_attach = multi + dev.lun_wwn = WWN + dev.alceml_bdev = dev.alceml_name = f"alceml_{DEV_ID}" + dev.pt_bdev = f"alceml_{DEV_ID}_PT" + dev.nvmf_nqn = multi_attach.device_nqn(f"{constants.CLUSTER_NQN}:cluster-1", DEV_ID) + dev.nvmf_ip = "10.0.0.1" + dev.nvmf_port = 4420 + dev.node_paths = {"node-a": {"path": BY_ID, "bdev": "aio_x"}} + a.nvme_devices = [dev] + for n in (b, c, d): + n.agent.lun_info.return_value = ([_lun_entry()], None) + a.status = StorageNode.STATUS_OFFLINE + return FakeDB(_cluster(plugin, multi), [a, b, c, d]), dev + + +class _TakeoverCase(unittest.TestCase): + + def setUp(self): + FakeArrayPlugin.reset() + array.register_plugin(PLUGIN, FakeArrayPlugin) + self.addCleanup(array.unregister_plugin, PLUGIN) + self.events = [] + for target, attr, kw in ( + (lun_takeover, "DBController", {}), + (lun_takeover.ec, "log_event_cluster", + {"side_effect": lambda **k: self.events.append(k)}), + (device_controller, "device_set_state", {"return_value": True})): + p = patch.object(target, attr, **kw) + setattr(self, f"m_{attr}", p.start()) + self.addCleanup(p.stop) + qos = patch("simplyblock_core.controllers.qos_controller.get_qos_weights_list", return_value=[]) + qos.start() + self.addCleanup(qos.stop) + + def run_takeover(self, db, failed="node-a", **kw): + self.m_DBController.return_value = db + return lun_takeover.take_over_lun_outcome(DEV_ID, failed, "test", **kw) + + def dev(self, db): + return db.get_storage_device_by_id(DEV_ID) + + +# ------------------------------------------------------------ the takeover + +class TestTakeover(_TakeoverCase): + + def test_happy_path(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + # 1. fence of the failed node, by hostname, first. + self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-a")]) + b = db.nodes["node-b"] + # 4. open-only alceml on the secondary, over the by-id path. + aio = b.spdk.ops("bdev_aio_create") + self.assertEqual(aio[0][1][1], BY_ID) + alceml = b.spdk.ops("bdev_alceml_create") + self.assertEqual(len(alceml), 1) + self.assertEqual(alceml[0][2]["pba_init_mode"], 1) + self.assertEqual(alceml[0][2]["stuck_io_timeout_ms"], constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS) + nqn = multi_attach.device_nqn(db.cluster.nqn, DEV_ID) + sub = b.spdk.ops("subsystem_create")[0] + self.assertEqual(sub[1][0], nqn) + self.assertEqual((sub[2]["min_cntlid"], sub[2]["max_cntlid"]), multi_attach.cntlid_range(1)) + ns = b.spdk.ops("nvmf_subsystem_add_ns")[0] + self.assertEqual(ns[1][2], multi_attach.namespace_uuid(self.dev(db))) + self.assertEqual(b.spdk.ops("listeners_create")[0][1][2], "10.0.0.2") + # 6. record. + dev = self.dev(db) + self.assertEqual(dev.serving_node(), "node-b") + self.assertEqual(dev.nvmf_ip, "10.0.0.2") + self.assertEqual(dev.node_paths["node-b"]["path"], BY_ID) + self.assertIn("node-a", dev.fenced_node_ids) + self.assertEqual(dev.status, NVMeDevice.STATUS_ONLINE) + self.m_device_set_state.assert_not_called() + # 5 + 7. new path on every ONLINE node (standby's loopback too), old one off. + for nid in ("node-b", "node-c", "node-d"): + spdk = db.nodes[nid].spdk + att = spdk.ops("attach") + self.assertEqual(len(att), 1, nid) + self.assertEqual(att[0][1][:5], (f"remote_alceml_{DEV_ID}", nqn, "10.0.0.2", 4420, "TCP")) + self.assertEqual(att[0][2]["multipath"], "multipath") + self.assertEqual(att[0][2]["ctrlr_loss_timeout_sec"], constants.MULTI_ATTACH_HOLD_SEC) + self.assertEqual(spdk.ops("policy")[0][1], (f"remote_alceml_{DEV_ID}n1", "active_passive")) + det = spdk.ops("detach") + self.assertEqual(det[0][2]["traddr"], "10.0.0.1") + self.assertEqual(db.nodes["node-a"].spdk.calls, []) + self.assertTrue(any("taken over" in e["message"] for e in self.events)) + + def test_never_formats(self): + db, _ = _bed() + self.run_takeover(db) + for n in db.nodes.values(): + for call in n.spdk.ops("bdev_alceml_create"): + self.assertEqual(call[2]["pba_init_mode"], lun_takeover.ALCEML_OPEN_ONLY) + + def test_fence_failure_means_no_takeover(self): + for mode in ("fail", "fail_falsy"): + with self.subTest(mode=mode): + FakeArrayPlugin.reset() + getattr(FakeArrayPlugin, mode).add("fence") + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(self.dev(db).serving_node(), "node-a") + for n in db.nodes.values(): + self.assertEqual(n.spdk.calls, []) + n.agent.lun_info.assert_not_called() + + def test_no_plugin_refuses(self): + db, _ = _bed(plugin="") + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(self.dev(db).serving_node(), "node-a") + self.assertTrue(all(n.spdk.calls == [] for n in db.nodes.values())) + self.assertTrue(any("no array plugin" in e["message"] for e in self.events)) + + def test_unknown_plugin_refuses(self): + db, _ = _bed(plugin="no-such-plugin") + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(FakeArrayPlugin.calls, []) + + def test_no_standby_can_read_holds(self): + db, _ = _bed() + for nid in ("node-b", "node-c"): + db.nodes[nid].agent.lun_info.return_value = ([_lun_entry(signature=None)], None) + self.assertEqual(self.run_takeover(db), lun_takeover.NO_STANDBY) + self.assertEqual(self.dev(db).serving_node(), "node-a") + self.assertTrue(all(n.spdk.calls == [] for n in db.nodes.values())) + # The node that is not owner/secondary/tertiary is never a candidate. + db.nodes["node-d"].agent.lun_info.assert_not_called() + self.assertTrue(any("no standby can read" in e["message"] for e in self.events)) + + def test_secondary_offline_uses_tertiary(self): + db, _ = _bed() + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + self.assertEqual(self.dev(db).nvmf_ip, "10.0.0.3") + self.assertEqual(db.nodes["node-b"].spdk.calls, []) + + def test_secondary_unreadable_uses_tertiary(self): + db, _ = _bed() + db.nodes["node-b"].agent.lun_info.return_value = ([], None) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + + def test_rerun_is_idempotent(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + for n in db.nodes.values(): + n.spdk.calls.clear() + FakeArrayPlugin.calls.clear() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(FakeArrayPlugin.calls, []) + b = db.nodes["node-b"] + self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) + self.assertEqual(b.spdk.ops("subsystem_create"), []) + self.assertEqual(self.dev(db).serving_node(), "node-b") + self.assertTrue(b.spdk.ops("attach")) + + def test_rerun_after_partial_export_reuses_it(self): + db, _ = _bed() + b = db.nodes["node-b"] + b.spdk.fail.add("nvmf_subsystem_add_ns") + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + # Cleanup closed what this attempt opened; nothing recorded. + self.assertEqual(self.dev(db).serving_node(), "node-a") + self.assertNotIn(f"alceml_{DEV_ID}", b.spdk.bdevs) + self.assertTrue(b.spdk.ops("bdev_alceml_delete")) + b.spdk.fail.clear() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + def test_concurrent_takeover_is_refused(self): + db, _ = _bed() + db.lock_holder = "another-runner" + self.assertEqual(self.run_takeover(db), lun_takeover.IN_PROGRESS) + self.assertEqual(FakeArrayPlugin.calls, []) + + def test_failed_standby_is_taken_over_again(self): + db, _ = _bed() + self.run_takeover(db) + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + FakeArrayPlugin.calls.clear() + self.assertEqual(self.run_takeover(db, failed="node-b"), lun_takeover.TAKEN_OVER) + self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-b")]) + dev = self.dev(db) + self.assertEqual(dev.serving_node(), "node-c") + self.assertEqual(sorted(dev.fenced_node_ids), ["node-a", "node-b"]) + + def test_unavailable_device_comes_back_online(self): + db, _ = _bed(dev_status=NVMeDevice.STATUS_UNAVAILABLE) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.m_device_set_state.assert_called_once_with( + DEV_ID, NVMeDevice.STATUS_ONLINE, cause=device_controller.CAUSE_LUN_TAKEOVER, + connect_peers=False) + + def test_unfence_node(self): + db, _ = _bed() + self.run_takeover(db) + self.assertTrue(lun_takeover.unfence_node(DEV_ID, "node-a")) + self.assertEqual(FakeArrayPlugin.calls[-1], ("unfence", WWN, "host-node-a")) + self.assertEqual(self.dev(db).fenced_node_ids, []) + # The serving node is never "unfenced" (it was never fenced). + self.assertFalse(lun_takeover.unfence_node(DEV_ID, "node-b")) + + def test_regular_device_is_refused(self): + db, _ = _bed(multi=False) + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(FakeArrayPlugin.calls, []) + + def test_size_mismatch_is_refused(self): + db, dev = _bed() + dev.size = SIZE * 2 + for nid in ("node-b", "node-c"): + db.nodes[nid].agent.lun_info.return_value = ([_lun_entry(size=SIZE * 2)], None) + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(self.dev(db).serving_node(), "node-a") + + +# ----------------------------------------------------------- helpers/array + +class TestHelpers(unittest.TestCase): + + def test_pick_lun_path(self): + dev = NVMeDevice() + dev.node_id = "a" + dev.size = SIZE + dev.node_paths = {"a": {"path": BY_ID}} + self.assertEqual(multi_attach.pick_lun_path([_lun_entry()], dev), (BY_ID, "")) + self.assertIsNone(multi_attach.pick_lun_path([], dev)[0]) + self.assertIsNone(multi_attach.pick_lun_path([_lun_entry(signature=False)], dev)[0]) + self.assertIsNone(multi_attach.pick_lun_path([_lun_entry(signature=None)], dev)[0]) + other = _lun_entry(path="/dev/disk/by-id/scsi-x") + self.assertIsNone(multi_attach.pick_lun_path([other, dict(other, name="sdd")], dev)[0]) + self.assertEqual(multi_attach.pick_lun_path([other, _lun_entry()], dev)[0], BY_ID) + + def test_array_host_and_candidates(self): + n = StorageNode() + n.uuid = "a" + n.hostname, n.rpc_port = "worker-1_5260", 5260 + self.assertEqual(multi_attach.array_host(n), "worker-1") + n.secondary_node_id, n.tertiary_node_id = "b", "c" + self.assertEqual(multi_attach.standby_candidates(n, "a"), ["b", "c"]) + self.assertEqual(multi_attach.standby_candidates(n, "b"), ["c", "a"]) + + def test_fake_plugin_is_not_registered_by_default(self): + with self.assertRaises(array.ArrayError): + array.plugin_class("fake") + with self.assertRaises(array.ArrayNotConfigured): + array.plugin_for_cluster(_cluster(plugin="")) + + def test_entry_point_loading(self): + ep = MagicMock(value="pkg:Cls") + ep.load.return_value = FakeArrayPlugin + eps = MagicMock() + eps.select.return_value = [ep] + with patch.object(array.metadata, "entry_points", return_value=eps): + plugin = array.load_plugin("vendor", "https://array", {"pool": "p"}) + eps.select.assert_called_once_with(group=array.ENTRY_POINT_GROUP, name="vendor") + self.assertIsInstance(plugin, FakeArrayPlugin) + self.assertEqual((plugin.endpoint, plugin.options), ("https://array", {"pool": "p"})) + ep.load.return_value = object + with patch.object(array.metadata, "entry_points", return_value=eps): + with self.assertRaises(array.ArrayError): + array.load_plugin("vendor") + + def test_find_lun_devices(self): + from simplyblock_web import node_utils + inv = [{"type": "disk", "name": "sdb", "device_path": "/dev/sdb", "wwn": "0X600A", + "serial": "s1", "size": 1, "by_id_path": "/x"}, + {"type": "disk", "name": "sdc", "device_path": "/dev/sdc", "wwn": "", "serial": "zz", + "size": 1, "by_id_path": "/y"}] + with patch.object(node_utils, "get_block_devices_info", return_value=inv), \ + patch.object(node_utils, "simplyblock_data_signature", return_value=True) as sig: + out = node_utils.find_lun_devices("600a") + self.assertEqual([e["name"] for e in out], ["sdb"]) + self.assertTrue(out[0]["has_alceml_signature"]) + sig.assert_called_once_with("sdb") + + +# ------------------------------------------------------------------ monitor + +class TestMonitorHook(unittest.TestCase): + + def _run(self, db): + started = [] + with patch.object(storage_node_monitor, "db", db), \ + patch.object(storage_node_monitor, "_start_lun_takeover", + side_effect=lambda did, nid, why: started.append((did, nid))), \ + patch.object(storage_node_monitor.device_controller, + "device_set_unavailable") as unavailable: + storage_node_monitor._set_node_devices_unavailable(db.nodes["node-a"], "test") + return started, [c.args[0] for c in unavailable.call_args_list] + + def test_multi_attach_starts_takeover_instead_of_unavailable(self): + db, dev = _bed() + plain = NVMeDevice() + plain.uuid, plain.node_id, plain.status = "plain", "node-a", NVMeDevice.STATUS_ONLINE + db.nodes["node-a"].nvme_devices.append(plain) + started, unavailable = self._run(db) + self.assertEqual(started, [(DEV_ID, "node-a")]) + self.assertEqual(unavailable, ["plain"]) + + def test_lun_served_elsewhere_is_untouched_and_taken_over_luns_follow(self): + db, dev = _bed() + dev.serving_node_id = "node-b" + foreign = copy.deepcopy(dev) + foreign.uuid, foreign.node_id, foreign.serving_node_id = "foreign", "node-c", "node-a" + db.nodes["node-c"].nvme_devices = [foreign] + started, unavailable = self._run(db) + self.assertEqual(started, [("foreign", "node-a")]) + self.assertEqual(unavailable, []) + + def test_regular_cluster_unchanged(self): + db, dev = _bed(multi=False) + started, unavailable = self._run(db) + self.assertEqual(started, []) + self.assertEqual(unavailable, [DEV_ID]) + + def test_worker_marks_unavailable_on_failure_only(self): + db, dev = _bed() + for outcome, expected in ((lun_takeover.FAILED, [DEV_ID]), + (lun_takeover.TAKEN_OVER, []), + (lun_takeover.IN_PROGRESS, [])): + with self.subTest(outcome=outcome), \ + patch.object(storage_node_monitor, "db", db), \ + patch.object(lun_takeover, "take_over_lun_outcome", return_value=outcome), \ + patch.object(storage_node_monitor.device_controller, + "device_set_unavailable") as unavailable: + storage_node_monitor._lun_takeover_worker(DEV_ID, "node-a", "t", 0.0) + self.assertEqual([c.args[0] for c in unavailable.call_args_list], expected) + + def test_worker_holds_until_the_hold_expires(self): + db, dev = _bed() + with patch.object(storage_node_monitor, "db", db), \ + patch.object(lun_takeover, "take_over_lun_outcome", return_value=lun_takeover.NO_STANDBY), \ + patch.object(storage_node_monitor.time, "monotonic", return_value=100.0), \ + patch.object(storage_node_monitor.time, "sleep") as sleep, \ + patch.object(storage_node_monitor.device_controller, + "device_set_unavailable") as unavailable: + storage_node_monitor._lun_takeover_worker(DEV_ID, "node-a", "t", 90.0) + sleep.assert_called_once_with(90.0 + constants.MULTI_ATTACH_HOLD_SEC - 100.0) + unavailable.assert_called_once_with(DEV_ID) + + +# ----------------------------------------------------- owner restart/discovery + +class TestOwnerReturn(unittest.TestCase): + + def test_restart_skips_luns_served_elsewhere(self): + db, dev = _bed() + dev.serving_node_id = "node-b" + own = copy.deepcopy(dev) + own.uuid, own.serving_node_id = "own-served", "" + a = db.nodes["node-a"] + a.nvme_devices = [dev, own] + a.jm_device = None + built = [] + with patch.object(storage_node_ops, "DBController"), \ + patch.object(storage_node_ops, "_create_storage_device_stack", + side_effect=lambda rpc, nvme, sn, ar: built.append(nvme.get_id()) or nvme), \ + patch.object(StorageNode, "write_to_db"): + self.assertTrue(storage_node_ops._prepare_cluster_devices_on_restart(a)) + self.assertEqual(built, ["own-served"]) + self.assertEqual([d.get_id() for d in a.nvme_devices], [DEV_ID, "own-served"]) + + def test_restart_skips_luns_it_is_fenced_from(self): + db, dev = _bed(dev_status=NVMeDevice.STATUS_UNAVAILABLE) + dev.fenced_node_ids = ["node-a"] + a = db.nodes["node-a"] + a.jm_device = None + with patch.object(storage_node_ops, "DBController"), patch.object(storage_node_ops, "_create_storage_device_stack") as build, patch.object(StorageNode, "write_to_db"): + self.assertTrue(storage_node_ops._prepare_cluster_devices_on_restart(a)) + build.assert_not_called() + + def _reconcile(self, db, node, found): + with patch.object(storage_node_ops, "DBController", return_value=db): + return storage_node_ops._reconcile_discovered_devices(node, found) + + def _found(self, serial, wwn=""): + d = NVMeDevice() + d.uuid, d.serial_number, d.lun_wwn = f"new-{serial}", serial, wwn + d.device_name = "sdx" + return d + + def test_discovery_never_adopts_another_nodes_lun(self): + db, dev = _bed() + b = db.nodes["node-b"] + b.nvme_devices = [] + new = self._reconcile(db, b, [self._found(WWN.upper(), WWN), self._found("own-lun")]) + self.assertEqual([d.serial_number for d in new], ["own-lun"]) + self.assertNotIn(WWN.upper(), [d.serial_number for d in b.nvme_devices]) + + def test_missing_lun_served_elsewhere_is_kept(self): + db, dev = _bed() + dev.serving_node_id = "node-b" + a = db.nodes["node-a"] + self.assertEqual(self._reconcile(db, a, []), []) + self.assertEqual(dev.status, NVMeDevice.STATUS_ONLINE) + + def test_regular_cluster_discovery_unchanged(self): + db, dev = _bed(multi=False) + b = db.nodes["node-b"] + b.nvme_devices = [] + new = self._reconcile(db, b, [self._found(WWN, WWN)]) + self.assertEqual(len(new), 1) + a = db.nodes["node-a"] + self.assertEqual(self._reconcile(db, a, []), []) + self.assertEqual(dev.status, NVMeDevice.STATUS_REMOVED) + + def test_add_node_drops_foreign_luns(self): + db, dev = _bed() + with patch.object(storage_node_ops, "DBController", return_value=db): + kept = storage_node_ops.drop_foreign_luns(db.nodes["node-b"], [self._found(WWN, WWN), + self._found("mine")]) + self.assertEqual([d.serial_number for d in kept], ["mine"]) + + +# --------------------------------------------------------- connect / state + +class TestConnectAndState(unittest.TestCase): + + def test_connect_uses_the_serving_node(self): + db, dev = _bed() + dev.serving_node_id = "node-b" + dev.nvmf_ip = "10.0.0.2" + db.nodes["node-a"].active_tcp = False # the dead owner: must not matter + db.nodes["node-b"].active_rdma = True + rpc = MagicMock() + rpc.bdev_nvme_controller_list.return_value = [] + rpc.bdev_get.return_value = {"name": "b"} + attach = MagicMock() + attach.bdev_nvme_attach_controller.return_value = [f"remote_alceml_{DEV_ID}n1"] + with patch.object(storage_node_ops, "DBController", return_value=db): + storage_node_ops._connect_device_attach( + f"remote_alceml_{DEV_ID}", dev, db.nodes["node-d"], rpc, attach, ["10.0.0.2"], True) + call = attach.bdev_nvme_attach_controller.call_args + self.assertEqual(call.args[2:5], ("10.0.0.2", 4420, "RDMA")) + rpc.bdev_nvme_set_multipath_policy.assert_called_once_with( + f"remote_alceml_{DEV_ID}n1", "active_passive") + + def test_connect_adds_the_missing_path_to_a_held_controller(self): + db, dev = _bed() + dev.serving_node_id = "node-b" + dev.nvmf_ip = "10.0.0.2" + held = [{"ctrlrs": [{"state": "resetting", "trid": {"traddr": "10.0.0.1"}}]}] + rpc = MagicMock() + rpc.bdev_nvme_controller_list.return_value = held + rpc.bdev_get.return_value = {"name": "b"} + attach = MagicMock() + attach.bdev_nvme_attach_controller.return_value = True + with patch.object(storage_node_ops, "DBController", return_value=db), \ + patch.object(storage_node_ops.time, "sleep") as sleep: + out = storage_node_ops._connect_device_attach( + f"remote_alceml_{DEV_ID}", dev, db.nodes["node-d"], rpc, attach, ["10.0.0.2"], True) + sleep.assert_not_called() + self.assertEqual(out, f"remote_alceml_{DEV_ID}n1") + self.assertEqual(attach.bdev_nvme_attach_controller.call_args.args[2], "10.0.0.2") + + def _set_online(self, db, cause): + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(device_controller, "_atomic_device_set"), \ + patch.object(device_controller.device_events, "device_status_change"), \ + patch.object(device_controller.distr_controller, "send_dev_status_event"): + return device_controller.device_set_state( + DEV_ID, NVMeDevice.STATUS_ONLINE, cause=cause, connect_peers=False) + + def test_online_while_owner_down_only_via_takeover(self): + db, dev = _bed(dev_status=NVMeDevice.STATUS_UNAVAILABLE) + dev.serving_node_id = "node-b" + self.assertFalse(self._set_online(db, device_controller.CAUSE_OTHER)) + self.assertTrue(self._set_online(db, device_controller.CAUSE_LUN_TAKEOVER)) + + def test_never_online_while_the_serving_node_is_down_or_fenced(self): + db, dev = _bed(dev_status=NVMeDevice.STATUS_UNAVAILABLE) + db.nodes["node-a"].status = StorageNode.STATUS_ONLINE + dev.serving_node_id = "node-b" + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + self.assertFalse(self._set_online(db, device_controller.CAUSE_NODE_RECOVERY)) + dev.serving_node_id = "" + dev.fenced_node_ids = ["node-a"] + self.assertFalse(self._set_online(db, device_controller.CAUSE_OTHER)) + dev.fenced_node_ids = [] + self.assertTrue(self._set_online(db, device_controller.CAUSE_OTHER)) + + def test_takeover_cause_needs_an_online_serving_node(self): + db, dev = _bed(dev_status=NVMeDevice.STATUS_UNAVAILABLE) + self.assertFalse(self._set_online(db, device_controller.CAUSE_LUN_TAKEOVER)) + dev.serving_node_id = "node-b" + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + self.assertFalse(self._set_online(db, device_controller.CAUSE_LUN_TAKEOVER)) + + +if __name__ == "__main__": + unittest.main() From 12cb0678ed75b34c10b728ab0dc54dd13358a6d4 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 12:57:43 +0100 Subject: [PATCH 09/50] feat(multi-attach): array LUN mode, phase 4 (array plugins, KMS credentials, AWS EBS io2) simplyblock now creates, multi-attaches, grows, fences and deletes the LUNs of a multi-attach cluster itself, through a vendor plugin. Every new path is gated on a multi-attach cluster with an array plugin (and, for LUN creation at node add, Cluster.array_lun_size); regular clusters are unchanged. - KMS: put_secret / get_secret / delete_secret on the KMS base, HCPClient (kv v2 on kv_mount) and LocalKMS (kv_store, prefix secrets/ apart from the DEK keys/); values in and out as SecretStr, plaintext only at the store. - array plugin interface: connect(endpoint, credentials, options), capabilities() dict, ensure_host, create_lun, attach, detach(force), resize (grow only), get, delete_lun, list_luns; fence / unfence default to a forced detach / attach. Every created LUN carries the simplyblock-cluster-id tag; untagged or foreign LUNs raise ForeignLunError. plugin_for_cluster connects with credentials read from the KMS (cluster//array/). FakeArrayPlugin implements it all in memory. Built-in fallback for the aws-ebs entry point. - registration: controllers/array_controller.register_array / get_arrays / delete_array; POST/GET/DELETE /api/v2/clusters/{id}/arrays; CLI cluster set-array (--credentials-file) / get-array / remove-array. The plugin is loaded, connected and its capabilities checked (max_attachments >= FTT+1) before anything is saved; credentials never reach the cluster record or a response. cluster_ops.set_array_plugin delegates to it. - aws-ebs plugin (simplyblock_core/array/aws_ebs.py, entry point aws-ebs): io2/io1 Multi-Attach volumes in options.az; LUN id = volume id, wwn = the NVMe serial vol (lsblk SERIAL); host = EC2 instance id; attach with a free device name, forced detach as fence (waits until the attachment is gone), modify_volume grow (waits for optimizing/completed), delete after detaching all. Access key / secret from the KMS or the instance role. Agent cloud info now carries availability_zone / region. - lifecycle: node add creates the node's LUN (ensure_host, create, attach, wait for it in the agent inventory) and continues as an lblk add with it (a lone local device becomes the journal); LVS role changes (activation, expansion, and a periodic pass from the health-check service) present each LUN to owner + secondary + tertiary + serving node, fill node_paths, and detach from nodes that lost the role - never the serving node, never re-attaching a fenced node; grow-devices --lun-factor grows every LUN at the array and waits until every node sees it (agent rescan now resizes dm-multipath maps); node delete detaches; cluster delete deletes the cluster's LUNs and the KMS secret. - takeover fences by NVMeDevice.array_lun_id / StorageNode.array_host_id when the plugin created them, else by WWN / hostname as in phase 3. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 5 + simplyblock_cli/cli-reference.yaml | 70 ++ simplyblock_cli/cli.py | 29 + simplyblock_cli/clibase.py | 54 +- simplyblock_core/array/__init__.py | 220 +++- simplyblock_core/array/aws_ebs.py | 405 +++++++ simplyblock_core/array/fake.py | 160 ++- simplyblock_core/cluster_ops.py | 48 +- .../controllers/array_controller.py | 735 ++++++++++++ .../controllers/device_controller.py | 25 +- .../controllers/events_controller.py | 2 + simplyblock_core/controllers/lun_takeover.py | 18 +- simplyblock_core/kms/__init__.py | 6 + simplyblock_core/kms/_base.py | 47 + simplyblock_core/kms/_fdb.py | 21 +- simplyblock_core/kms/_hcp.py | 35 +- simplyblock_core/models/cluster.py | 14 +- simplyblock_core/models/nvme_device.py | 11 +- simplyblock_core/models/storage_node.py | 4 + simplyblock_core/multi_attach.py | 20 +- .../services/health_check_service.py | 8 + simplyblock_core/storage_node_ops.py | 24 + simplyblock_core/utils/__init__.py | 5 + simplyblock_web/api/internal/_node_info.py | 4 + simplyblock_web/api/v2/cluster/__init__.py | 16 +- simplyblock_web/api/v2/cluster/array.py | 79 ++ simplyblock_web/node_utils.py | 50 +- tests/unit/test_array_lifecycle.py | 1016 +++++++++++++++++ .../unit/web/api/v2/test_device_endpoints.py | 17 + 29 files changed, 3043 insertions(+), 105 deletions(-) create mode 100644 simplyblock_core/array/aws_ebs.py create mode 100644 simplyblock_core/controllers/array_controller.py create mode 100644 simplyblock_web/api/v2/cluster/array.py create mode 100644 tests/unit/test_array_lifecycle.py diff --git a/pyproject.toml b/pyproject.toml index 1aba7329d1..fd720737c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,11 @@ sbcli-dev = "simplyblock_cli.cli:main" simplyblock-service = "simplyblock_core.services.__main__:main" simplyblock-task-runner = "simplyblock_core.services.task_runners:main" +# Storage-array plugins of the multi-attach array LUN mode +# (simplyblock_core.array); the name is the value of Cluster.array_plugin. +[project.entry-points."simplyblock.array_plugins"] +aws-ebs = "simplyblock_core.array.aws_ebs:AwsEbsPlugin" + [dependency-groups] test = [ "mock", diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 684fe06c75..a4103645dc 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1554,6 +1554,76 @@ commands: type: bool default: false action: store_true + - name: "--lun-factor" + help: > + Multi-attach cluster with a registered array only: first grow every LUN the array plugin created by + this factor (greater than 1, e.g. 1.5) at the array and wait until every node sees the new size. + dest: lun_factor + type: str + - name: set-array + help: "Registers the storage array of a multi-attach cluster (or updates it, e.g. to rotate credentials)." + usage: > + simplyblock creates, multi-attaches, grows, fences and deletes the LUNs of a multi-attach cluster through + the array plugin registered here. The plugin is loaded and connected with the credentials before anything + is saved. The credentials are stored in the cluster's KMS, never in the cluster record. With --lun-size, + every node added afterwards gets a LUN of that size created and attached on the array. + arguments: + - name: "cluster_id" + help: "The cluster id." + dest: cluster_id + type: str + completer: _completer_get_cluster_list + - name: "--plugin" + help: "The array plugin, e.g. `aws-ebs`." + dest: plugin + type: str + required: true + - name: "--endpoint" + help: "The array's management endpoint (for aws-ebs: an EC2 endpoint URL override, usually empty)." + dest: endpoint + type: str + default: "" + - name: "--options" + help: 'Plugin options as a JSON object, e.g. `{"az": "us-east-1a", "iops": 6000}`.' + dest: options + type: str + - name: "--credentials-file" + help: > + A JSON file with the array credentials as an object of strings (for aws-ebs: aws_access_key_id, + aws_secret_access_key; omit to use the instance role). + dest: credentials_file + type: str + - name: "--lun-size" + help: "Size of the LUN created for each node added to the cluster, e.g. `1T`. Default: none." + dest: lun_size + type: size + default: "0" + - name: get-array + help: "Shows the storage array registered for a multi-attach cluster (without its credentials)." + arguments: + - name: "cluster_id" + help: "The cluster id." + dest: cluster_id + type: str + completer: _completer_get_cluster_list + - name: remove-array + help: "Removes the storage array registration of a multi-attach cluster and its credentials." + arguments: + - name: "cluster_id" + help: "The cluster id." + dest: cluster_id + type: str + completer: _completer_get_cluster_list + - name: "array_id" + help: "The array id." + dest: array_id + type: str + - name: "--force" + help: "Remove it even though LUNs created through it exist (they can then no longer be fenced)." + dest: force + type: bool + default: false + action: store_true - name: show help: "Shows a cluster's statistics." arguments: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 33a97664bb..2fa4ec45d3 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -382,6 +382,9 @@ def init_cluster(self): self.init_cluster__status(subparser) self.init_cluster__complete_expand(subparser) self.init_cluster__grow_devices(subparser) + self.init_cluster__set_array(subparser) + self.init_cluster__get_array(subparser) + self.init_cluster__remove_array(subparser) self.init_cluster__show(subparser) self.init_cluster__get(subparser) if self.developer_mode: @@ -568,6 +571,26 @@ def init_cluster__grow_devices(self, subparser): subcommand = self.add_sub_command(subparser, 'grow-devices', 'Grows every Linux block device (lblk) of the cluster online to the size of its block device.') subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list subcommand.add_argument('--no-rebalance', help='Keep the device weights after an uneven growth and do not rebalance the cluster.', default=False, dest='no_rebalance', action='store_true') + subcommand.add_argument('--lun-factor', help='Multi-attach cluster with a registered array only: first grow every LUN the array plugin created by this factor (greater than 1, e.g. 1.5) at the array and wait until every node sees the new size.', type=str, dest='lun_factor') + + def init_cluster__set_array(self, subparser): + subcommand = self.add_sub_command(subparser, 'set-array', 'Registers the storage array of a multi-attach cluster (or updates it, e.g. to rotate credentials).') + subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list + subcommand.add_argument('--plugin', help='The array plugin, e.g. `aws-ebs`.', type=str, dest='plugin', required=True) + subcommand.add_argument('--endpoint', help='The array\'s management endpoint (for aws-ebs: an EC2 endpoint URL override, usually empty).', type=str, default='', dest='endpoint') + subcommand.add_argument('--options', help='Plugin options as a JSON object, e.g. `{"az": "us-east-1a", "iops": 6000}`.', type=str, dest='options') + subcommand.add_argument('--credentials-file', help='A JSON file with the array credentials as an object of strings (for aws-ebs: aws_access_key_id, aws_secret_access_key; omit to use the instance role).', type=str, dest='credentials_file') + subcommand.add_argument('--lun-size', help='Size of the LUN created for each node added to the cluster, e.g. `1T`. Default: none.', type=size_type(), default='0', dest='lun_size') + + def init_cluster__get_array(self, subparser): + subcommand = self.add_sub_command(subparser, 'get-array', 'Shows the storage array registered for a multi-attach cluster (without its credentials).') + subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list + + def init_cluster__remove_array(self, subparser): + subcommand = self.add_sub_command(subparser, 'remove-array', 'Removes the storage array registration of a multi-attach cluster and its credentials.') + subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list + subcommand.add_argument('array_id', help='The array id.', type=str) + subcommand.add_argument('--force', help='Remove it even though LUNs created through it exist (they can then no longer be fenced).', default=False, dest='force', action='store_true') def init_cluster__show(self, subparser): subcommand = self.add_sub_command(subparser, 'show', 'Shows a cluster\'s statistics.') @@ -1719,6 +1742,12 @@ def run(self): ret = self.cluster__complete_expand(sub_command, args) elif sub_command in ['grow-devices']: ret = self.cluster__grow_devices(sub_command, args) + elif sub_command in ['set-array']: + ret = self.cluster__set_array(sub_command, args) + elif sub_command in ['get-array']: + ret = self.cluster__get_array(sub_command, args) + elif sub_command in ['remove-array']: + ret = self.cluster__remove_array(sub_command, args) elif sub_command in ['show']: ret = self.cluster__show(sub_command, args) elif sub_command in ['get']: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index dae719fb56..680d4f77d6 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -8,11 +8,14 @@ from pathlib import Path from typing import Any +from pydantic import SecretStr + from simplyblock_cli.alerting_config_parser import parse_alerting_config from simplyblock_core import cluster_ops, constants, db_controller, utils from simplyblock_core import mgmt_node_ops as mgmt_ops from simplyblock_core import storage_node_ops as storage_ops from simplyblock_core.controllers import ( + array_controller, device_controller, fdb_backup_controller, health_controller, @@ -804,10 +807,20 @@ def cluster__change_name(self, sub_command, args): return True def cluster__grow_devices(self, sub_command, args): + lun_factor = getattr(args, 'lun_factor', None) + if lun_factor is not None: + try: + lun_factor = float(lun_factor) + except ValueError: + raise ValueError(f"--lun-factor must be a number, not {lun_factor!r}") + extra = {"lun_factor": lun_factor} if lun_factor is not None else {} result = device_controller.grow_cluster_devices( - args.cluster_id, rebalance=not getattr(args, 'no_rebalance', False)) - lines = [f"{r['device_id']}: {utils.humanbytes(r['old_size'])} -> {utils.humanbytes(r['new_size'])}" - for r in result["devices"] if r["resized"]] + args.cluster_id, rebalance=not getattr(args, 'no_rebalance', False), **extra) + lines = [f"LUN {r['lun_id']} of {dev_id}: {utils.humanbytes(r['old_size'])} -> " + f"{utils.humanbytes(r['new_size'])} at the array" + for dev_id, r in (result.get("luns") or {}).items()] + lines += [f"{r['device_id']}: {utils.humanbytes(r['old_size'])} -> {utils.humanbytes(r['new_size'])}" + for r in result["devices"] if r["resized"]] lines += [f"{dev_id}: failed: {err}" for dev_id, err in result["errors"].items()] rebalance = result.get("rebalance") if rebalance and rebalance["started"]: @@ -824,6 +837,41 @@ def cluster__grow_devices(self, sub_command, args): "grow-devices without --no-rebalance has updated the weights.") return "\n".join(lines) or "No device has grown" + def cluster__set_array(self, sub_command, args): + options = {} + if args.options: + try: + options = json.loads(args.options) + except json.JSONDecodeError as e: + raise ValueError(f"--options is not valid JSON: {e}") + if not isinstance(options, dict): + raise ValueError("--options must be a JSON object") + credentials = {} + if args.credentials_file: + try: + raw = json.loads(Path(args.credentials_file).read_text()) + except (OSError, json.JSONDecodeError) as e: + # Never the file content: it is the secret. + raise ValueError(f"--credentials-file could not be read as JSON: {type(e).__name__}") + if not isinstance(raw, dict) or not all(isinstance(v, str) for v in raw.values()): + raise ValueError("--credentials-file must hold a JSON object of strings") + # Wrapped at the boundary; unwrapped only by the KMS and the plugin. + credentials = {str(k): SecretStr(v) for k, v in raw.items()} + view = array_controller.register_array( + args.cluster_id, args.plugin, args.endpoint or "", options, credentials, + int(args.lun_size or 0)) + return _format_result([view], json=getattr(args, "json", False)) + + def cluster__get_array(self, sub_command, args): + arrays = array_controller.get_arrays(args.cluster_id) + if not arrays: + return "No array registered" + return _format_result(arrays, json=getattr(args, "json", False)) + + def cluster__remove_array(self, sub_command, args): + array_controller.delete_array(args.cluster_id, args.array_id, force=args.force) + return True + def cluster__complete_expand(self, sub_command, args): cluster_ops.cluster_expand(args.cluster_id) return True diff --git a/simplyblock_core/array/__init__.py b/simplyblock_core/array/__init__.py index 35721d9d9e..9077b90974 100644 --- a/simplyblock_core/array/__init__.py +++ b/simplyblock_core/array/__init__.py @@ -6,11 +6,14 @@ host multipath layer may still hold queued writes, and once released they would land after, and overwrite, newer data written through the new node. -This package is the minimal interface the takeover needs (phase 3): an -abstract :class:`ArrayPlugin` with ``fence`` / ``unfence`` / ``capabilities``, -and a registry that loads a vendor plugin by name from the entry-point group -``simplyblock.array_plugins``. Credentials (KMS) and further operations are -phase 4. +This package holds the plugin interface (:class:`ArrayPlugin`: LUN create / +attach / detach / resize / get / delete / list and fence / unfence) and a +registry that loads a vendor plugin by name from the entry-point group +``simplyblock.array_plugins``. The array's credentials live in the KMS at +``cluster//array/``; the Cluster record keeps the plugin +name, endpoint and options only. Built-in plugin: ``aws-ebs`` +(:mod:`.aws_ebs`). Registration and the LUN lifecycle are in +:mod:`simplyblock_core.controllers.array_controller`. Plugins are found only through the entry-point group or an explicit :func:`register_plugin` call. The test double in :mod:`.fake` is never @@ -31,6 +34,12 @@ #: name is the value of ``Cluster.array_plugin``. ENTRY_POINT_GROUP = "simplyblock.array_plugins" +#: Plugins shipped with simplyblock (also declared as entry points in +#: pyproject.toml). +BUILTIN_PLUGINS = { + "aws-ebs": "simplyblock_core.array.aws_ebs:AwsEbsPlugin", +} + class ArrayError(Exception): """An array operation failed, or no plugin could be loaded.""" @@ -40,36 +49,147 @@ class ArrayNotConfigured(ArrayError): """The cluster names no array plugin.""" +#: Tag every LUN a plugin creates carries, with the cluster id as its value. +#: A plugin acts only on LUNs carrying its cluster's tag (ForeignLunError). +TAG_CLUSTER_ID = "simplyblock-cluster-id" +#: Tag carrying the LUN name the control plane asked for. +TAG_LUN_NAME = "simplyblock-lun-name" + + +class ForeignLunError(ArrayError): + """The LUN was not created by this cluster's plugin: it carries no, or + another, cluster tag. It is never touched.""" + + class ArrayPlugin(abc.ABC): """The array operations the control plane needs. - ``lun_wwn`` is the LUN's world-wide name as the hosts report it - (``NVMeDevice.lun_wwn``); ``host`` is the storage node's identity on the - array (see ``multi_attach.array_host``). Every operation is idempotent: a - fence of a host that is already fenced succeeds. A method reports failure - by raising (any exception, preferably :class:`ArrayError`) or returning a - falsy value; the caller treats both the same way. + simplyblock creates, presents (multi-attaches), grows, fences and deletes + the LUNs of a multi-attach cluster itself, through one plugin per vendor. + + Identities: ``lun_id`` is the array's own id of a LUN, as returned by + :meth:`create_lun` (``NVMeDevice.array_lun_id``); a plugin should also + accept the LUN's ``wwn`` as the hosts report it where it can map it + (phase 3 fenced by ``NVMeDevice.lun_wwn``). ``host_id`` is the array's id + of a storage node's host, as returned by :meth:`ensure_host` + (``StorageNode.array_host_id``). + + Every operation is idempotent: creating a LUN that exists returns it, + attaching an attached LUN, detaching a detached one and fencing a fenced + host succeed. A method reports failure by raising (preferably + :class:`ArrayError`); ``fence`` / ``unfence`` may also return a falsy + value. A plugin acts only on LUNs it created for ``cluster_id``: it tags + each with :data:`TAG_CLUSTER_ID` and raises :class:`ForeignLunError` for + any other LUN. + + Credentials (``connect``) are a flat dict of ``SecretStr`` (from the + KMS); a plugin unwraps them only where it hands them to its SDK and never + logs them. """ - def __init__(self, endpoint: str = "", options: dict | None = None, credentials=None): + def __init__(self, endpoint: str = "", options: dict | None = None, credentials=None, + cluster_id: str = ""): self.endpoint = endpoint self.options = dict(options or {}) - # Phase 4 resolves these from the KMS; a placeholder until then. - self.credentials = credentials + self.credentials = dict(credentials or {}) + self.cluster_id = cluster_id + + def __repr__(self): + # Never the credentials. + return f"{type(self).__name__}(endpoint={self.endpoint!r}, cluster_id={self.cluster_id!r})" + + def connect(self, endpoint: str, credentials: dict, options: dict) -> None: + """Set up the session to the array. Called once per instance, before + any other operation; raises when the array cannot be reached or the + credentials are refused. The default only stores the arguments.""" + self.endpoint = endpoint or "" + self.credentials = dict(credentials or {}) + self.options = dict(options or {}) + + def capabilities(self) -> dict: + """``{"protocols": [...], "supports_reservations": bool, + "max_attachments": int}`` (0 = no limit known).""" + return {"protocols": [], "supports_reservations": False, "max_attachments": 0} + + # ------------------------------------------------------------ ownership + + def cluster_tags(self, tags: dict | None = None, name: str = "") -> dict: + """``tags`` plus the ownership tags every created LUN carries.""" + if not self.cluster_id: + raise ArrayError("the plugin has no cluster id; it cannot tag what it creates") + out = {str(k): str(v) for k, v in (tags or {}).items()} + out[TAG_CLUSTER_ID] = self.cluster_id + if name: + out[TAG_LUN_NAME] = name + return out + + def check_owned(self, lun_id: str, tags: dict | None) -> None: + """Raise :class:`ForeignLunError` unless ``tags`` (the LUN's tags on the + array) carry this cluster's id.""" + if not self.cluster_id or (tags or {}).get(TAG_CLUSTER_ID) != self.cluster_id: + raise ForeignLunError( + f"LUN {lun_id} was not created by simplyblock cluster {self.cluster_id or '?'}; " + "it is not touched") + + # ---------------------------------------------------------- operations + + @abc.abstractmethod + def ensure_host(self, name: str, initiators: dict) -> str: + """The array's id of the host ``name`` with ``initiators`` (vendor + specific: NQNs, WWPNs, IQNs, an EC2 instance id), created when the + array keeps host objects.""" + + @abc.abstractmethod + def create_lun(self, name: str, size_bytes: int, tags: dict) -> dict: + """Create LUN ``name`` (at least ``size_bytes``) tagged with ``tags`` + plus :meth:`cluster_tags`, or return the existing one of that name. + Returns ``{"id", "wwn", "size"}``; ``wwn`` is the identity the hosts + report for it (lsblk WWN or SERIAL).""" + + @abc.abstractmethod + def attach(self, lun_id: str, host_id: str) -> None: + """Present the LUN to the host (multi-attach: in addition to the + hosts it is presented to already).""" + + @abc.abstractmethod + def detach(self, lun_id: str, host_id: str, force: bool = False) -> None: + """Stop presenting the LUN to the host. ``force``: also when the host + does not cooperate; returns only once the host can no longer reach + the LUN.""" + + @abc.abstractmethod + def resize(self, lun_id: str, size_bytes: int) -> None: + """Grow the LUN to at least ``size_bytes``; returns once the hosts can + see the new size (a rescan away). Shrinking is refused.""" + + @abc.abstractmethod + def get(self, lun_id: str) -> dict: + """``{"id", "wwn", "size", "hosts": [host ids it is presented to]}``.""" @abc.abstractmethod - def fence(self, lun_wwn: str, host: str) -> bool: - """Revoke ``host``'s access to the LUN so no I/O of it, including I/O - queued in its multipath layer, can reach the LUN any more. Returns - only once the array has applied it.""" + def delete_lun(self, lun_id: str) -> None: + """Delete the LUN, detaching it from every host first. Deleting a LUN + that is gone succeeds.""" @abc.abstractmethod - def unfence(self, lun_wwn: str, host: str) -> bool: - """Give ``host`` access to the LUN again.""" + def list_luns(self) -> list[dict]: + """Every LUN of this cluster (tagged with its id), as :meth:`get` + returns them.""" + + # ------------------------------------------------------------- fencing - def capabilities(self) -> set[str]: - """Names of the optional operations the plugin supports (phase 4).""" - return {"fence", "unfence"} + def fence(self, lun_id: str, host_id: str) -> bool: + """Revoke the host's access to the LUN so no I/O of it, including + I/O queued in its multipath layer, can reach the LUN any more. + Returns only once the array has applied it. Default: a forced + detach.""" + self.detach(lun_id, host_id, force=True) + return True + + def unfence(self, lun_id: str, host_id: str) -> bool: + """Give the host access to the LUN again. Default: attach.""" + self.attach(lun_id, host_id) + return True _registry: dict[str, type[ArrayPlugin]] = {} @@ -110,6 +230,10 @@ def plugin_class(name: str) -> type[ArrayPlugin]: if cls is not None: return cls ep = _entry_point(name) + if ep is None and name in BUILTIN_PLUGINS: + # The package's own entry point, when it runs from a source tree that + # was never installed (entry points come from installed metadata). + ep = metadata.EntryPoint(name=name, value=BUILTIN_PLUGINS[name], group=ENTRY_POINT_GROUP) if ep is None: raise ArrayError(f"no array plugin {name!r} in entry-point group {ENTRY_POINT_GROUP}") try: @@ -122,32 +246,66 @@ def plugin_class(name: str) -> type[ArrayPlugin]: def load_plugin(name: str, endpoint: str = "", options: dict | None = None, - credentials=None) -> ArrayPlugin: - """An instance of plugin ``name`` for ``endpoint``. + credentials=None, cluster_id: str = "") -> ArrayPlugin: + """A connected instance of plugin ``name`` for ``endpoint``. Raises: ArrayNotConfigured: ``name`` is empty. - ArrayError: the plugin cannot be found, loaded or constructed. + ArrayError: the plugin cannot be found, loaded, constructed or + connected. """ if not name: raise ArrayNotConfigured("no array plugin is configured") cls = plugin_class(name) try: - return cls(endpoint=endpoint, options=options, credentials=credentials) + plugin = cls(endpoint=endpoint, options=options, credentials=credentials, + cluster_id=cluster_id) except Exception as e: raise ArrayError(f"array plugin {name!r} could not be set up: {e}") from e + try: + plugin.connect(endpoint, dict(credentials or {}), dict(options or {})) + except Exception as e: + # An SDK error message does not carry the credentials, and they are + # never formatted into one here. + raise ArrayError(f"array plugin {name!r} could not connect to {endpoint or 'the array'}: {e}") from e + return plugin + + +def array_credentials(cluster) -> dict: + """The credentials of the cluster's array from the KMS (``SecretStr`` + values), or ``{}`` when the array was registered without any (e.g. the + EBS plugin on the instance role) or no array is registered. + + Raises: + ArrayError: a secret is recorded but cannot be read. + """ + array_id = getattr(cluster, "array_id", "") or "" + if not array_id or getattr(cluster, "array_has_credentials", False) is not True: + return {} + from simplyblock_core import kms + try: + with kms.create_kms_connection(cluster) as conn: + return conn.get_secret(kms.array_secret_path(cluster.get_id(), array_id)) + except Exception as e: + raise ArrayError(f"array credentials could not be read from the KMS: {e}") from e def plugin_for_cluster(cluster) -> ArrayPlugin: - """The array plugin a cluster is configured with (``Cluster.array_plugin`` - / ``array_endpoint`` / ``array_options``). + """The connected array plugin a cluster is configured with + (``Cluster.array_plugin`` / ``array_endpoint`` / ``array_options``, + credentials from the KMS). Raises: ArrayNotConfigured: the cluster names no plugin. - ArrayError: it cannot be loaded. + ArrayError: it cannot be loaded or connected. """ + name = getattr(cluster, "array_plugin", "") or "" + if not name: + raise ArrayNotConfigured("no array plugin is configured") return load_plugin( - getattr(cluster, "array_plugin", "") or "", + name, getattr(cluster, "array_endpoint", "") or "", getattr(cluster, "array_options", None) or {}, + credentials=array_credentials(cluster), + cluster_id=cluster.get_id(), ) diff --git a/simplyblock_core/array/aws_ebs.py b/simplyblock_core/array/aws_ebs.py new file mode 100644 index 0000000000..3e43c4a58b --- /dev/null +++ b/simplyblock_core/array/aws_ebs.py @@ -0,0 +1,405 @@ +"""AWS EBS io2 Multi-Attach as the "array" of a multi-attach cluster. + +Each storage node's LUN is an io2 (or io1) volume created with +``MultiAttachEnabled``, attached to the owner and to its LVS secondary and +tertiary. All instances must be Nitro instances in the one availability zone +of the cluster (``options["az"]``); a multi-attach volume cannot leave its AZ. + +Identities: + +* LUN id: the volume id (``vol-0123...``). The host-side identity (``wwn``) + is the NVMe serial EBS gives the volume's namespace, ``vol0123...`` (the id + without its dash), which lsblk reports as SERIAL on every instance the + volume is attached to; ``NVMeDevice.lun_wwn`` and the agent's + ``find_lun_devices`` match on it. Both forms are accepted wherever a LUN id + is expected. +* host id: the EC2 instance id (``ensure_host`` takes + ``{"ec2_instance_id": ...}``; the node agent reports it from the instance + metadata service as ``cloud_instance.id``, StorageNode.cloud_instance_id). + +Ownership: every volume created here is tagged ``simplyblock-cluster-id = +``; a volume without this cluster's tag is never attached, +detached, resized or deleted (:class:`ForeignLunError`). + +Fencing: ``fence`` is a forced ``DetachVolume`` of the failed instance's +attachment, and returns only once EC2 reports the attachment gone. A +detached EBS volume is hot-removed from the instance's PCIe bus; the instance +can issue no further I/O to it, queued or new. + +Credentials (KMS secret, all optional): ``aws_access_key_id``, +``aws_secret_access_key``, ``aws_session_token``. Without them the default +boto3 chain applies (the control plane's instance role / IRSA). + +Options: ``az`` (required), ``region`` (default: ``az`` without its letter), +``volume_type`` (``io2`` default, or ``io1``), ``iops`` (default 3000), +``encrypted`` (default true), ``kms_key_id``, ``tags`` (dict, added to every +volume), ``poll_interval`` (s, default 2), ``timeout`` (s, default 300). +``endpoint`` (the registration's endpoint) is an EC2 endpoint URL override +(VPC endpoint), empty for the regional default. +""" + +from __future__ import annotations + +import logging +import math +import time + +from pydantic import SecretStr + +from simplyblock_core.array import TAG_CLUSTER_ID, TAG_LUN_NAME, ArrayError, ArrayPlugin + +logger = logging.getLogger() + +GIB = 1024 ** 3 +#: Smallest io1/io2 volume. +MIN_SIZE_GIB = 4 +#: EBS Multi-Attach: at most 16 Nitro instances per volume. +MAX_ATTACHMENTS = 16 +MULTI_ATTACH_TYPES = ("io2", "io1") +_LIVE_ATTACHMENT = ("attaching", "attached", "busy") +_GONE_VOLUME = ("deleting", "deleted", "error") + + +def volume_wwn(volume_id: str) -> str: + """The NVMe serial of an EBS volume as an instance sees it.""" + return volume_id.replace("-", "", 1) + + +def volume_id_of(lun: str) -> str: + """``vol-0abc`` from either the volume id or its NVMe serial.""" + lun = (lun or "").strip() + if lun.startswith("vol-"): + return lun + if lun.startswith("vol") and len(lun) > 3: + return "vol-" + lun[3:] + raise ArrayError(f"{lun!r} is no EBS volume id") + + +def _plain(value) -> str: + if isinstance(value, SecretStr): + return value.get_secret_value() + return str(value or "") + + +class AwsEbsPlugin(ArrayPlugin): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ec2 = None + self._sleep = time.sleep + + # ------------------------------------------------------------- session + + def connect(self, endpoint, credentials, options): + super().connect(endpoint, credentials, options) + az = self.options.get("az") or "" + if not az: + raise ArrayError("aws-ebs: option 'az' (the availability zone of every node) is required") + vtype = self.options.get("volume_type", "io2") + if vtype not in MULTI_ATTACH_TYPES: + raise ArrayError(f"aws-ebs: volume_type must be one of {MULTI_ATTACH_TYPES} (Multi-Attach)") + try: + import boto3 + except ImportError as e: # pragma: no cover - boto3 is a dependency + raise ArrayError("aws-ebs: boto3 is not installed") from e + creds = self.credentials + key_id = _plain(creds.get("aws_access_key_id")) + secret = _plain(creds.get("aws_secret_access_key")) + if bool(key_id) != bool(secret): + raise ArrayError("aws-ebs: aws_access_key_id and aws_secret_access_key go together") + session = boto3.session.Session( + aws_access_key_id=key_id or None, + aws_secret_access_key=secret or None, + aws_session_token=_plain(creds.get("aws_session_token")) or None, + region_name=self.region, + ) + self.ec2 = session.client("ec2", endpoint_url=endpoint or None) + # Validates the credentials and the zone in one call. + try: + zones = self.ec2.describe_availability_zones(ZoneNames=[az])["AvailabilityZones"] + except Exception as e: + raise ArrayError(f"aws-ebs: availability zone {az} not usable: {self._err(e)}") from e + if not zones: + raise ArrayError(f"aws-ebs: availability zone {az} not found") + + @property + def region(self) -> str: + return self.options.get("region") or (self.options.get("az") or "")[:-1] + + @property + def _poll(self) -> float: + return float(self.options.get("poll_interval", 2)) + + @property + def _timeout(self) -> float: + return float(self.options.get("timeout", 300)) + + @staticmethod + def _err(e) -> str: + resp = getattr(e, "response", None) or {} + err = resp.get("Error") or {} + if err: + return f"{err.get('Code')}: {err.get('Message')}" + return str(e) + + @staticmethod + def _code(e) -> str: + return ((getattr(e, "response", None) or {}).get("Error") or {}).get("Code", "") + + def _client(self): + if self.ec2 is None: + raise ArrayError("aws-ebs: not connected") + return self.ec2 + + def capabilities(self): + return {"protocols": ["nvme"], "supports_reservations": True, + "max_attachments": MAX_ATTACHMENTS} + + # ------------------------------------------------------------- helpers + + def _describe(self, volume_id): + try: + vols = self._client().describe_volumes(VolumeIds=[volume_id])["Volumes"] + except Exception as e: + if self._code(e) == "InvalidVolume.NotFound": + return None + raise ArrayError(f"aws-ebs: describe {volume_id}: {self._err(e)}") from e + return vols[0] if vols else None + + @staticmethod + def _tags(vol) -> dict: + return {t["Key"]: t["Value"] for t in vol.get("Tags") or []} + + def _owned(self, lun): + vid = volume_id_of(lun) + vol = self._describe(vid) + if vol is None or vol.get("State") in _GONE_VOLUME: + raise ArrayError(f"aws-ebs: volume {vid} not found") + self.check_owned(vid, self._tags(vol)) + return vol + + @staticmethod + def _attachment(vol, instance_id): + for att in vol.get("Attachments") or []: + if att.get("InstanceId") == instance_id: + return att + return None + + @staticmethod + def _summary(vol) -> dict: + return { + "id": vol["VolumeId"], + "wwn": volume_wwn(vol["VolumeId"]), + "size": int(vol.get("Size") or 0) * GIB, + "hosts": [a["InstanceId"] for a in vol.get("Attachments") or [] + if a.get("State") in _LIVE_ATTACHMENT], + "state": vol.get("State", ""), + } + + def _wait(self, what, check): + """Poll ``check()`` until it returns a truthy value or the timeout.""" + deadline = time.monotonic() + self._timeout + while True: + result = check() + if result: + return result + if time.monotonic() >= deadline: + raise ArrayError(f"aws-ebs: timed out waiting for {what}") + self._sleep(self._poll) + + def _free_device_name(self, instance_id) -> str: + try: + res = self._client().describe_instances(InstanceIds=[instance_id]) + inst = res["Reservations"][0]["Instances"][0] + except Exception as e: + raise ArrayError(f"aws-ebs: describe instance {instance_id}: {self._err(e)}") from e + used = {m.get("DeviceName") for m in inst.get("BlockDeviceMappings") or []} + used.add(inst.get("RootDeviceName")) + # The device name is ignored for NVMe on Nitro but must be unique + # per instance. /dev/sd[f-z], then /dev/sd[f-z][a-z]. + letters = "fghijklmnopqrstuvwxyz" + candidates = [f"/dev/sd{a}" for a in letters] + candidates += [f"/dev/sd{a}{b}" for a in letters for b in "abcdefghijklmnopqrstuvwxyz"] + for name in candidates: + if name not in used: + return name + raise ArrayError(f"aws-ebs: no free device name on {instance_id}") + + # ---------------------------------------------------------- operations + + def ensure_host(self, name, initiators): + instance_id = (initiators or {}).get("ec2_instance_id") or "" + if not instance_id.startswith("i-"): + raise ArrayError(f"aws-ebs: host {name} has no EC2 instance id") + try: + res = self._client().describe_instances(InstanceIds=[instance_id]) + inst = res["Reservations"][0]["Instances"][0] + except ArrayError: + raise + except Exception as e: + raise ArrayError(f"aws-ebs: instance {instance_id} of host {name}: {self._err(e)}") from e + az = (inst.get("Placement") or {}).get("AvailabilityZone", "") + if az != self.options.get("az"): + raise ArrayError( + f"aws-ebs: instance {instance_id} of host {name} is in {az}, the volumes in " + f"{self.options.get('az')}; Multi-Attach volumes cannot cross zones") + return instance_id + + def create_lun(self, name, size_bytes, tags): + all_tags = self.cluster_tags(dict(self.options.get("tags") or {}, **(tags or {})), name) + all_tags.setdefault("Name", name) + try: + found = self._client().describe_volumes(Filters=[ + {"Name": f"tag:{TAG_CLUSTER_ID}", "Values": [self.cluster_id]}, + {"Name": f"tag:{TAG_LUN_NAME}", "Values": [name]}, + ])["Volumes"] + except Exception as e: + raise ArrayError(f"aws-ebs: look up volume {name}: {self._err(e)}") from e + live = [v for v in found if v.get("State") not in _GONE_VOLUME] + if len(live) > 1: + raise ArrayError(f"aws-ebs: {len(live)} volumes are named {name} in this cluster") + if live: + vol = live[0] + logger.info("aws-ebs: volume %s for %s exists", vol["VolumeId"], name) + else: + size_gib = max(MIN_SIZE_GIB, math.ceil(int(size_bytes) / GIB)) + kwargs = { + "AvailabilityZone": self.options["az"], + "Size": size_gib, + "VolumeType": self.options.get("volume_type", "io2"), + "Iops": int(self.options.get("iops", 3000)), + "MultiAttachEnabled": True, + "Encrypted": bool(self.options.get("encrypted", True)), + "TagSpecifications": [{ + "ResourceType": "volume", + "Tags": [{"Key": k, "Value": v} for k, v in sorted(all_tags.items())], + }], + } + if self.options.get("kms_key_id"): + kwargs["KmsKeyId"] = self.options["kms_key_id"] + try: + vol = self._client().create_volume(**kwargs) + except Exception as e: + raise ArrayError(f"aws-ebs: create volume {name}: {self._err(e)}") from e + logger.info("aws-ebs: created volume %s (%s GiB) for %s", vol["VolumeId"], size_gib, name) + vid = vol["VolumeId"] + + def _available(): + v = self._describe(vid) + if v is not None and v.get("State") == "error": + raise ArrayError(f"aws-ebs: volume {vid} failed to create") + return v if v is not None and v.get("State") in ("available", "in-use") else None + vol = self._wait(f"volume {vid} to become available", _available) + return {k: v for k, v in self._summary(vol).items() if k in ("id", "wwn", "size")} + + def attach(self, lun_id, host_id): + vol = self._owned(lun_id) + vid = vol["VolumeId"] + att = self._attachment(vol, host_id) + if att is None or att.get("State") not in _LIVE_ATTACHMENT: + live = [a for a in vol.get("Attachments") or [] if a.get("State") in _LIVE_ATTACHMENT] + if len(live) >= MAX_ATTACHMENTS: + raise ArrayError(f"aws-ebs: volume {vid} is attached to {MAX_ATTACHMENTS} instances") + device = self._free_device_name(host_id) + try: + self._client().attach_volume(VolumeId=vid, InstanceId=host_id, Device=device) + except Exception as e: + raise ArrayError(f"aws-ebs: attach {vid} to {host_id}: {self._err(e)}") from e + logger.info("aws-ebs: attaching %s to %s as %s", vid, host_id, device) + + def _attached(): + a = self._attachment(self._describe(vid) or {}, host_id) + return a is not None and a.get("State") == "attached" + self._wait(f"volume {vid} to attach to {host_id}", _attached) + + def detach(self, lun_id, host_id, force=False): + vol = self._owned(lun_id) + vid = vol["VolumeId"] + att = self._attachment(vol, host_id) + if att is None or att.get("State") == "detached": + return + if att.get("State") != "detaching" or force: + try: + self._client().detach_volume(VolumeId=vid, InstanceId=host_id, Force=bool(force)) + except Exception as e: + if self._code(e) not in ("IncorrectState", "InvalidAttachment.NotFound"): + raise ArrayError(f"aws-ebs: detach {vid} from {host_id}: {self._err(e)}") from e + logger.info("aws-ebs: detaching %s from %s%s", vid, host_id, " (forced)" if force else "") + + def _detached(): + v = self._describe(vid) + a = self._attachment(v or {}, host_id) + return a is None or a.get("State") == "detached" + self._wait(f"volume {vid} to detach from {host_id}", _detached) + + def resize(self, lun_id, size_bytes): + vol = self._owned(lun_id) + vid = vol["VolumeId"] + target = math.ceil(int(size_bytes) / GIB) + current = int(vol.get("Size") or 0) + if target < current: + raise ArrayError(f"aws-ebs: volume {vid} is {current} GiB; shrinking to {target} GiB is refused") + if target > current: + try: + self._client().modify_volume(VolumeId=vid, Size=target) + except Exception as e: + raise ArrayError(f"aws-ebs: grow {vid} to {target} GiB: {self._err(e)}") from e + logger.info("aws-ebs: growing %s from %s to %s GiB", vid, current, target) + + def _grown(): + try: + mods = self._client().describe_volumes_modifications( + VolumeIds=[vid])["VolumesModifications"] + except Exception as e: + if self._code(e) == "InvalidVolumeModification.NotFound": + mods = [] + else: + raise ArrayError(f"aws-ebs: modification of {vid}: {self._err(e)}") from e + for m in mods: + state = m.get("ModificationState") + if state == "failed": + raise ArrayError(f"aws-ebs: growing {vid} failed: {m.get('StatusMessage', '')}") + if state == "modifying": + return False + # "optimizing" or "completed": the new size is usable. + return True + self._wait(f"volume {vid} to grow", _grown) + + def get(self, lun_id): + return self._summary(self._owned(lun_id)) + + def delete_lun(self, lun_id): + vid = volume_id_of(lun_id) + vol = self._describe(vid) + if vol is None or vol.get("State") in ("deleting", "deleted"): + return + self.check_owned(vid, self._tags(vol)) + for att in vol.get("Attachments") or []: + if att.get("State") in _LIVE_ATTACHMENT or att.get("State") == "detaching": + self.detach(vid, att["InstanceId"], force=True) + + def _available(): + v = self._describe(vid) + return v is None or v.get("State") == "available" + self._wait(f"volume {vid} to become available", _available) + try: + self._client().delete_volume(VolumeId=vid) + except Exception as e: + if self._code(e) != "InvalidVolume.NotFound": + raise ArrayError(f"aws-ebs: delete {vid}: {self._err(e)}") from e + logger.info("aws-ebs: deleted volume %s", vid) + + def list_luns(self): + if not self.cluster_id: + raise ArrayError("aws-ebs: no cluster id to list the volumes of") + out = [] + try: + pages = self._client().get_paginator("describe_volumes").paginate( + Filters=[{"Name": f"tag:{TAG_CLUSTER_ID}", "Values": [self.cluster_id]}]) + for page in pages: + for vol in page["Volumes"]: + if vol.get("State") not in _GONE_VOLUME: + out.append(self._summary(vol)) + except Exception as e: + raise ArrayError(f"aws-ebs: list volumes: {self._err(e)}") from e + return out diff --git a/simplyblock_core/array/fake.py b/simplyblock_core/array/fake.py index 1d30916e21..4986e25004 100644 --- a/simplyblock_core/array/fake.py +++ b/simplyblock_core/array/fake.py @@ -5,23 +5,34 @@ import threading from typing import ClassVar -from simplyblock_core.array import ArrayError, ArrayPlugin +from simplyblock_core.array import ArrayError, ArrayPlugin, ForeignLunError class FakeArrayPlugin(ArrayPlugin): - """Records every call in ``calls`` (``(op, lun_wwn, host)``) and the - fenced pairs in ``fenced``. ``fail`` = a set of operation names - (``"fence"``, ``"unfence"``) that raise :class:`ArrayError`; - ``fail_falsy`` the ones that return False instead. + """An array kept in memory. - State is per class (``reset()`` clears it), because the takeover builds a - fresh instance from the cluster record each time.""" + ``calls`` records every operation as ``(op, *args)``; ``fence`` / + ``unfence`` as ``(op, lun, host)`` and the fenced pairs in ``fenced``. + ``luns``: lun id -> ``{"id", "name", "wwn", "size", "tags", "hosts"}``; + ``hosts``: host id -> ``{"name", "initiators"}``. ``fail`` = operation + names that raise :class:`ArrayError`; ``fail_falsy`` the ones that return + False instead (fence / unfence only). ``connected`` holds the + ``(endpoint, credentials, options)`` of every ``connect`` (not in + ``calls``). + + State is per class (``reset()`` clears it), because the control plane + builds a fresh instance from the cluster record each time.""" calls: ClassVar[list] = [] fenced: ClassVar[set] = set() fail: ClassVar[set] = set() fail_falsy: ClassVar[set] = set() - _lock = threading.Lock() + luns: ClassVar[dict] = {} + hosts: ClassVar[dict] = {} + connected: ClassVar[list] = [] + max_attachments: ClassVar[int] = 16 + _lock = threading.RLock() + _next: ClassVar[int] = 0 @classmethod def reset(cls): @@ -30,23 +41,138 @@ def reset(cls): cls.fenced = set() cls.fail = set() cls.fail_falsy = set() + cls.luns = {} + cls.hosts = {} + cls.connected = [] + cls.max_attachments = 16 + cls._next = 0 + + def _record(self, op, *args): + with self._lock: + type(self).calls.append((op, *args)) + if op in self.fail: + raise ArrayError(f"fake array: {op} {args} failed") + + def _lun(self, lun_id): + lun = self.luns.get(lun_id) + if lun is None: + # Accept the WWN as well, as the EBS plugin does. + lun = next((x for x in self.luns.values() if x["wwn"] == lun_id), None) + if lun is None: + raise ArrayError(f"fake array: no LUN {lun_id}") + self.check_owned(lun_id, lun["tags"]) + return lun + + def connect(self, endpoint, credentials, options): + super().connect(endpoint, credentials, options) + if "connect" in self.fail: + raise ArrayError("fake array: connect refused") + with self._lock: + type(self).connected.append((endpoint, dict(credentials or {}), dict(options or {}))) + + def capabilities(self): + return {"protocols": ["fake"], "supports_reservations": False, + "max_attachments": self.max_attachments} + + def ensure_host(self, name, initiators): + self._record("ensure_host", name) + host_id = f"host-{name}" + with self._lock: + type(self).hosts[host_id] = {"name": name, "initiators": dict(initiators or {})} + return host_id + + def create_lun(self, name, size_bytes, tags): + self._record("create_lun", name, size_bytes) + all_tags = self.cluster_tags(tags, name) + with self._lock: + for lun in self.luns.values(): + if lun["name"] == name and lun["tags"].get("simplyblock-cluster-id") == self.cluster_id: + return {"id": lun["id"], "wwn": lun["wwn"], "size": lun["size"]} + type(self)._next += 1 + lun_id = f"lun-{self._next}" + type(self).luns[lun_id] = {"id": lun_id, "name": name, "wwn": f"wwn{self._next:04d}", + "size": int(size_bytes), "tags": all_tags, "hosts": []} + return {"id": lun_id, "wwn": f"wwn{self._next:04d}", "size": int(size_bytes)} + + def add_foreign_lun(self, lun_id, wwn="foreign", tags=None): + """A LUN on the array that this cluster did not create (tests).""" + with self._lock: + type(self).luns[lun_id] = {"id": lun_id, "name": lun_id, "wwn": wwn, "size": 1, + "tags": dict(tags or {}), "hosts": []} + + def attach(self, lun_id, host_id): + self._record("attach", lun_id, host_id) + with self._lock: + lun = self._lun(lun_id) + if host_id not in lun["hosts"]: + if len(lun["hosts"]) >= self.max_attachments: + raise ArrayError(f"fake array: {lun_id} is attached to {self.max_attachments} hosts") + lun["hosts"].append(host_id) + + def detach(self, lun_id, host_id, force=False): + self._record("detach", lun_id, host_id, force) + with self._lock: + lun = self._lun(lun_id) + if host_id in lun["hosts"]: + lun["hosts"].remove(host_id) + + def resize(self, lun_id, size_bytes): + self._record("resize", lun_id, size_bytes) + with self._lock: + lun = self._lun(lun_id) + if int(size_bytes) < lun["size"]: + raise ArrayError(f"fake array: {lun_id} cannot shrink") + lun["size"] = int(size_bytes) + + def get(self, lun_id): + with self._lock: + lun = self._lun(lun_id) + return {"id": lun["id"], "wwn": lun["wwn"], "size": lun["size"], + "hosts": list(lun["hosts"])} + + def delete_lun(self, lun_id): + self._record("delete_lun", lun_id) + with self._lock: + try: + self._lun(lun_id) + except ForeignLunError: + raise + except ArrayError: + return + type(self).luns.pop(lun_id, None) + + def list_luns(self): + with self._lock: + return [{"id": x["id"], "wwn": x["wwn"], "size": x["size"], "hosts": list(x["hosts"])} + for x in self.luns.values() + if x["tags"].get("simplyblock-cluster-id") == self.cluster_id] + + # fence / unfence: recorded as before (phase 3 tests fence LUNs by WWN + # that were never created here); applied to a LUN the fake knows. - def _op(self, op, lun_wwn, host): + def _fence_op(self, op, lun, host): with self._lock: - type(self).calls.append((op, lun_wwn, host)) + type(self).calls.append((op, lun, host)) if op in self.fail: - raise ArrayError(f"fake array: {op} of {host} on {lun_wwn} failed") + raise ArrayError(f"fake array: {op} of {host} on {lun} failed") if op in self.fail_falsy: return False with self._lock: + known = lun in self.luns or any(x["wwn"] == lun for x in self.luns.values()) + if known: + entry = self._lun(lun) + if op == "fence" and host in entry["hosts"]: + entry["hosts"].remove(host) + elif op == "unfence" and host not in entry["hosts"]: + entry["hosts"].append(host) if op == "fence": - type(self).fenced.add((lun_wwn, host)) + type(self).fenced.add((lun, host)) else: - type(self).fenced.discard((lun_wwn, host)) + type(self).fenced.discard((lun, host)) return True - def fence(self, lun_wwn, host): - return self._op("fence", lun_wwn, host) + def fence(self, lun_id, host_id): + return self._fence_op("fence", lun_id, host_id) - def unfence(self, lun_wwn, host): - return self._op("unfence", lun_wwn, host) + def unfence(self, lun_id, host_id): + return self._fence_op("unfence", lun_id, host_id) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 2900d4aa13..a149b7b45c 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -37,6 +37,7 @@ ) from simplyblock_core.controllers import ( activation_gate, + array_controller, cluster_events, device_controller, pool_controller, @@ -1024,32 +1025,23 @@ def mutate(fresh): db_controller.atomic_update(cluster, mutate) -def set_array_plugin(cl_id, plugin: str, endpoint: str = "", options: dict | None = None) -> None: - """Configure the storage-array plugin of a multi-attach cluster (the - plugin that fences a failed node from its LUNs before a takeover; see - simplyblock_core.array). An empty ``plugin`` clears it, after which - takeovers are refused. Credentials come from the KMS (later phase). +def set_array_plugin(cl_id, plugin: str, endpoint: str = "", options: dict | None = None, + credentials: dict | None = None, lun_size: int = 0) -> None: + """Configure the storage-array plugin of a multi-attach cluster (see + controllers/array_controller.register_array: the plugin is connected + before anything is saved, credentials go to the KMS). An empty + ``plugin`` removes the registration, after which takeovers are refused. Raises: - ValueError: not a multi-attach cluster, or the plugin cannot be loaded. + ValueError: not a multi-attach cluster, the plugin cannot be loaded + or connected, or (removal) LUNs created through it exist. """ - from simplyblock_core import array, multi_attach cluster = db_controller.get_cluster_by_id(cl_id) - if not multi_attach.is_multi_attach(cluster): - raise ValueError("an array plugin is only used by a multi-attach cluster") - if plugin: - try: - array.plugin_class(plugin) - except array.ArrayError as e: - raise ValueError(str(e)) from e - opts = dict(options or {}) - - def mutate(fresh): - fresh.array_plugin = plugin or "" - fresh.array_endpoint = endpoint or "" - fresh.array_options = opts - - db_controller.atomic_update(cluster, mutate) + if not plugin: + if cluster.array_plugin: + array_controller.delete_array(cl_id, cluster.array_id) + return + array_controller.register_array(cl_id, plugin, endpoint, options, credentials, lun_size) def _wait_for_full_device_connectivity(cl_id, timeout_sec=300, poll_sec=10, checkpoint=None): @@ -1999,6 +1991,10 @@ def _on_site(msg: str, site=site) -> str: snode = db_controller.get_storage_node_by_id(snode.get_id()) used_nodes_as_tertiary.append(snode.tertiary_node_id) + # Multi-attach cluster with an array: present every owner's LUN to + # its new secondary / tertiary (a no-op on every other cluster). + array_controller.reconcile_after_role_change(cl_id) + # Sync replication: every LVS also gets its remote triplet on the other # site. Refs a re-activation finds are kept; missing ones are picked. if cluster.sync_replication: @@ -2785,6 +2781,10 @@ def cluster_expand(cl_id) -> None: sec_node_2.lvstore_stack_tertiary = snode.get_id() sec_node_2.write_to_db() + if cluster.ha_type == "ha": + # Multi-attach cluster with an array: the new roles' LUN paths. + array_controller.reconcile_after_role_change(cl_id) + if cluster.ha_type == "ha" and cluster.sync_replication: try: storage_node_ops.assign_remote_triplets(cluster, [snode.get_id()]) @@ -4436,6 +4436,10 @@ def delete_cluster(cl_id, force=False) -> None: if pools: raise ValueError("Can only remove Empty cluster, Pools found") + # Multi-attach cluster with an array: the only place its LUNs are + # deleted, with the array credentials in the KMS. No-op otherwise. + array_controller.delete_cluster_luns(cluster, force=force) + logger.info(f"Deleting Cluster {cl_id}") cluster_events.cluster_delete(cluster) cluster.remove(db_controller.kv_store) diff --git a/simplyblock_core/controllers/array_controller.py b/simplyblock_core/controllers/array_controller.py new file mode 100644 index 0000000000..5734228855 --- /dev/null +++ b/simplyblock_core/controllers/array_controller.py @@ -0,0 +1,735 @@ +"""Multi-attach array LUN mode, phase 4: the storage array of a cluster. + +simplyblock creates, presents, grows, fences and deletes the LUNs of a +multi-attach cluster itself, through the cluster's array plugin +(simplyblock_core.array). This module holds: + +* registration (``register_array`` / ``get_arrays`` / ``delete_array``): the + plugin is loaded, connected and asked for its capabilities before anything + is saved; credentials go to the KMS at ``cluster//array/``, + plugin name, endpoint and options to the Cluster record. One array per + cluster. +* the LUN lifecycle, every hook a no-op unless the cluster is multi-attach + and has an array plugin (``array_configured``), and acting only on LUNs the + plugin created (``NVMeDevice.array_lun_id``): + + - node add: ``provision_node_lun`` creates the node's LUN + (``Cluster.array_lun_size``), registers the host, attaches the LUN and + waits for it in the node agent's block-device inventory; the add then + continues as a regular lblk add with that device. + - LVS roles: ``reconcile_lun_attachments`` presents every owner's LUN to + its secondary and tertiary (and the node serving it), records the path + there in ``node_paths``, and detaches it from a node that lost its role + -- never from the serving node, never to a node fenced from it. + - growth: ``grow_luns`` grows every LUN by one factor at the array and + waits until every node it is presented to sees the new size. + - node delete: ``release_node_luns``; cluster delete: + ``delete_cluster_luns`` (the only place a LUN is deleted). +""" + +from __future__ import annotations + +import logging +import math +import threading +import time +import uuid as uuid_mod + +from pydantic import SecretStr + +from simplyblock_core import array, multi_attach + +logger = logging.getLogger() + +#: How long a node add waits for its new LUN to show in the agent inventory. +LUN_APPEAR_TIMEOUT_SEC = 120 +#: How long the attachment reconcile waits for a LUN on a newly attached node. +LUN_PATH_TIMEOUT_SEC = 30 +#: How long a growth waits for every node to see the new size. +LUN_GROW_VISIBLE_TIMEOUT_SEC = 180 +POLL_SEC = 3 + +_STATUS_REMOVED = "removed" +_STATUS_ONLINE = "online" + + +class ArrayLifecycleError(Exception): + """A LUN lifecycle step failed; the message says which.""" + + +def _sleep(sec): + time.sleep(sec) + + +def _db(): + from simplyblock_core.db_controller import DBController + return DBController() + + +def array_configured(cluster) -> bool: + """A multi-attach cluster with an array plugin: the lifecycle hooks act.""" + return multi_attach.is_multi_attach(cluster) and bool(getattr(cluster, "array_plugin", "")) + + +def provisions_luns(cluster) -> bool: + """Node adds create the node's LUN (an array is configured and a LUN + size set).""" + return array_configured(cluster) and int(getattr(cluster, "array_lun_size", 0) or 0) > 0 + + +def _event(cluster_id, message, error=False): + try: + from simplyblock_core.controllers import events_controller as ec + from simplyblock_core.models.events import EventObj + ec.log_event_cluster( + cluster_id=cluster_id, domain=ec.DOMAIN_CLUSTER, event=ec.EVENT_ARRAY_LUN, + db_object=_db().get_cluster_by_id(cluster_id), caused_by=ec.CAUSED_BY_MONITOR, + message=message, event_level=EventObj.LEVEL_ERROR if error else EventObj.LEVEL_INFO) + except Exception: # noqa: BLE001 - an event is best effort + logger.exception("could not log the array event") + + +# ------------------------------------------------------------ registration + +def _wrap_credentials(credentials) -> dict: + out = {} + for k, v in (credentials or {}).items(): + if not isinstance(k, str) or not k: + raise ValueError("credential names must be non-empty strings") + if isinstance(v, SecretStr): + out[k] = v + elif isinstance(v, str): + out[k] = SecretStr(v) + else: + raise ValueError(f"credential {k!r} must be a string") + return out + + +def _managed_devices(nodes): + """``(owner, device)`` of every LUN the array plugin created.""" + return [(n, d) for n in nodes for d in (n.nvme_devices or []) + if multi_attach.is_multi_attach(d) and getattr(d, "array_lun_id", "")] + + +def array_view(cluster, capabilities=None) -> dict: + """The registered array of ``cluster`` as the API shows it: everything + but the credentials (only whether there are any).""" + out = { + "id": cluster.array_id, + "plugin": cluster.array_plugin, + "endpoint": cluster.array_endpoint, + "options": dict(cluster.array_options or {}), + "lun_size": int(cluster.array_lun_size or 0), + "has_credentials": bool(cluster.array_has_credentials), + } + if capabilities is not None: + out["capabilities"] = capabilities + return out + + +def register_array(cluster_id, plugin, endpoint="", options=None, credentials=None, + lun_size=0) -> dict: + """Register (or re-register, e.g. to rotate credentials) the storage + array of a multi-attach cluster. + + The plugin is loaded and connected with the given credentials and its + capabilities read before anything is saved. Credentials (a flat dict of + strings / SecretStr) are stored in the KMS; the Cluster record keeps the + plugin name, endpoint, options and LUN size. Returns the array view + (``array_view``) with the plugin's capabilities. + + Raises: + ValueError: not a multi-attach cluster, the plugin cannot be loaded + or connected, its capabilities do not fit the cluster, or LUNs + created through another plugin exist. + """ + from simplyblock_core import kms + db = _db() + cluster = db.get_cluster_by_id(cluster_id) + if not multi_attach.is_multi_attach(cluster): + raise ValueError("an array is only used by a multi-attach cluster") + if not plugin: + raise ValueError("an array needs a plugin") + if int(lun_size or 0) < 0: + raise ValueError("lun_size must not be negative") + creds = _wrap_credentials(credentials) + opts = dict(options or {}) + nodes = db.get_storage_nodes_by_cluster_id(cluster_id) + if (cluster.array_plugin and cluster.array_plugin != plugin + and _managed_devices(nodes)): + raise ValueError( + f"LUNs created through plugin {cluster.array_plugin!r} exist; the plugin cannot change") + try: + inst = array.load_plugin(plugin, endpoint, opts, credentials=creds, cluster_id=cluster_id) + caps = dict(inst.capabilities() or {}) + except array.ArrayError as e: + raise ValueError(str(e)) from e + except Exception as e: # noqa: BLE001 - a vendor plugin + raise ValueError(f"array plugin {plugin!r} failed: {e}") from e + max_att = int(caps.get("max_attachments") or 0) + # Owner, secondary, tertiary: FTT + 1 hosts per LUN. + needed = int(getattr(cluster, "max_fault_tolerance", 1) or 1) + 1 + if max_att and max_att < needed: + raise ValueError(f"the array presents a LUN to at most {max_att} hosts; the cluster needs {needed}") + + array_id = cluster.array_id or str(uuid_mod.uuid4()) + path = kms.array_secret_path(cluster_id, array_id) + with kms.create_kms_connection(cluster) as conn: + if creds: + conn.put_secret(path, creds) + elif cluster.array_has_credentials: + conn.delete_secret(path) + + def mutate(fresh): + fresh.array_id = array_id + fresh.array_plugin = plugin + fresh.array_endpoint = endpoint or "" + fresh.array_options = opts + fresh.array_has_credentials = bool(creds) + fresh.array_lun_size = int(lun_size or 0) + + try: + db.atomic_update(cluster, mutate) + except Exception: + if creds and not cluster.array_has_credentials: + try: + with kms.create_kms_connection(cluster) as conn: + conn.delete_secret(path) + except Exception: # noqa: BLE001 + logger.exception("could not remove the array secret after a failed registration") + raise + cluster = db.get_cluster_by_id(cluster_id) + logger.info("Cluster %s: array %s registered (plugin %s, endpoint %s)", + cluster_id, array_id, plugin, endpoint or "-") + return array_view(cluster, caps) + + +def get_arrays(cluster_id) -> list[dict]: + """The registered arrays of a cluster (zero or one), without credentials.""" + cluster = _db().get_cluster_by_id(cluster_id) + if not cluster.array_plugin: + return [] + return [array_view(cluster)] + + +def delete_array(cluster_id, array_id, force=False) -> None: + """Remove the cluster's array registration and its credentials. + + Refused while LUNs created through it exist (a takeover could no longer + fence), unless ``force``. + + Raises: + KeyError: no array ``array_id``. + ValueError: LUNs of the array exist. + """ + from simplyblock_core import kms + db = _db() + cluster = db.get_cluster_by_id(cluster_id) + if not cluster.array_plugin or (cluster.array_id or "") != (array_id or ""): + raise KeyError(f"array {array_id} not found") + managed = _managed_devices(db.get_storage_nodes_by_cluster_id(cluster_id)) + if managed and not force: + raise ValueError(f"{len(managed)} LUN(s) created through the array exist; " + "takeovers would lose their fence") + if cluster.array_id and cluster.array_has_credentials: + with kms.create_kms_connection(cluster) as conn: + conn.delete_secret(kms.array_secret_path(cluster_id, cluster.array_id)) + + def mutate(fresh): + fresh.array_id = "" + fresh.array_plugin = "" + fresh.array_endpoint = "" + fresh.array_options = {} + fresh.array_has_credentials = False + fresh.array_lun_size = 0 + + db.atomic_update(cluster, mutate) + logger.info("Cluster %s: array %s removed", cluster_id, array_id) + + +# ------------------------------------------------------------------- hosts + +def host_initiators(hostname="", cloud_instance=None, system_id="") -> dict: + """What identifies a host on an array: its hostname, system id and, on + AWS, the EC2 instance id (reported by the node agent from the instance + metadata service, IMDSv2).""" + out = {"hostname": hostname} + if system_id: + out["system_id"] = system_id + cloud = cloud_instance or {} + if cloud.get("cloud") == "amazon" and str(cloud.get("id", "")).startswith("i-"): + out["ec2_instance_id"] = cloud["id"] + return out + + +def _node_initiators(node) -> dict: + cloud = {"cloud": getattr(node, "cloud_name", ""), "id": getattr(node, "cloud_instance_id", "")} + return host_initiators(multi_attach.array_host(node), cloud, getattr(node, "system_uuid", "")) + + +def ensure_node_host(plugin, node, db=None) -> str: + """The node's host id on the array, registered and recorded on the node + (``StorageNode.array_host_id``) the first time.""" + host_id = getattr(node, "array_host_id", "") or "" + if host_id: + return host_id + host_id = plugin.ensure_host(multi_attach.array_host(node), _node_initiators(node)) + if not host_id: + raise ArrayLifecycleError(f"the array returned no host id for node {node.get_id()}") + db = db or _db() + + def _set(n): + n.array_host_id = host_id + db.atomic_update(node, _set) + node.array_host_id = host_id + return host_id + + +# ---------------------------------------------------------------- node add + +def lun_name(hostname, socket) -> str: + return f"simplyblock-{hostname}-s{socket if socket is not None else 0}" + + +def _find_in_inventory(inventory, wwn): + want = multi_attach.normalize_wwn(wwn) + for d in inventory or []: + if d.get("type", "disk") != "disk": + continue + if want in (multi_attach.normalize_wwn(d.get("wwn", "")), + multi_attach.normalize_wwn(d.get("serial", ""))): + return d + return None + + +def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configured, + timeout=LUN_APPEAR_TIMEOUT_SEC): + """Create (or find) the LUN of the node being added, present it to the + node's host and wait until the host sees it. + + Returns ``(entry, host_id)``: ``entry`` is an lblk config entry for the + LUN (identified by its serial; carries ``lun_wwn`` and ``array_lun_id``) + to add to the node's ``lblk_devices``, ``host_id`` the host's id on the + array. A configured entry that already carries an ``array_lun_id`` (a + retried add) is reused. + + Raises: + ArrayLifecycleError: any step failed. + """ + hostname = node_info.get("hostname", "") + socket = node_config.get("socket") + try: + plugin = array.plugin_for_cluster(cluster) + host_id = plugin.ensure_host(hostname, host_initiators( + hostname, node_info.get("cloud_instance"), node_info.get("system_id", ""))) + existing = next((e for e in lblk_configured or [] if e.get("array_lun_id")), None) + if existing: + lun = {"id": existing["array_lun_id"], "wwn": existing.get("lun_wwn", "")} + else: + lun = plugin.create_lun(lun_name(hostname, socket), int(cluster.array_lun_size), + {"simplyblock-host": hostname, + "simplyblock-socket": str(socket if socket is not None else 0)}) + plugin.attach(lun["id"], host_id) + except array.ArrayError as e: + raise ArrayLifecycleError(f"array: {e}") from e + except Exception as e: # noqa: BLE001 - a vendor plugin + raise ArrayLifecycleError(f"array plugin failed: {e}") from e + logger.info("Node %s socket %s: LUN %s (%s) attached as host %s", + hostname, socket, lun["id"], lun["wwn"], host_id) + + deadline = time.monotonic() + timeout + while True: + inventory, err = snode_api.get_blockdevices() + found = _find_in_inventory(inventory, lun["wwn"]) + if found: + break + if time.monotonic() >= deadline: + raise ArrayLifecycleError( + f"LUN {lun['id']} ({lun['wwn']}) did not appear on {hostname} within {timeout}s" + + (f": {err}" if err else "")) + _sleep(POLL_SEC) + entry = { + "name": found["name"], + "serial": found.get("serial") or lun["wwn"], + "size": int(found.get("size") or 0), + "by_id": found.get("by_id_path", ""), + "numa": int(found.get("numa_node", -1)), + "model": found.get("model", ""), + "type": "disk", + "lun_wwn": lun["wwn"], + "array_lun_id": lun["id"], + } + return entry, host_id + + +def add_lun_to_config(lblk_configured, entry) -> list: + """``lblk_configured`` with the LUN entry in place of an older entry for + the same LUN (same serial or array id). + + On a multi-attach cluster a node's local devices can only hold its + journal (a data device must be reachable from the standbys). When the + host config names exactly one local device and flags none as the + journal, that device is flagged, so the journal is never placed on the + LUN by the smallest-device fallback.""" + out = [dict(e) for e in lblk_configured or [] + if e.get("serial") != entry["serial"] + and not (e.get("array_lun_id") and e.get("array_lun_id") == entry["array_lun_id"])] + local = [e for e in out if not e.get("array_lun_id")] + if len(local) == 1 and not any(e.get("journal") for e in out): + local[0]["journal"] = True + out.append(entry) + return out + + +# ------------------------------------------------------------- attachments + +def _lun_path_on(node, dev, timeout): + """The path of the device's LUN on ``node`` from its agent, or "".""" + owner_path = (dev.node_paths or {}).get(dev.node_id, {}).get("path", "") + deadline = time.monotonic() + timeout + while True: + try: + entries, _ = node.client(timeout=15, retry=1).lun_info(dev.lun_wwn) + except Exception as e: # noqa: BLE001 + logger.warning("lun_info on %s: %s", node.get_id(), e) + entries = None + for e in entries or []: + if owner_path and (owner_path in (e.get("by_id_paths") or []) or owner_path == e.get("by_id_path")): + return owner_path + if entries: + e = entries[0] + return e.get("by_id_path") or e.get("device_path") or "" + if time.monotonic() >= deadline: + return "" + _sleep(POLL_SEC) + + +def desired_hosts(owner, dev, nodes_by_id) -> set: + """Node ids the LUN of ``dev`` must be presented to: its owner, the + owner's LVS secondary and tertiary, and the node serving it -- minus + removed nodes and nodes fenced from it (the serving node always stays).""" + serving = dev.serving_node() + want = {owner.get_id(), getattr(owner, "secondary_node_id", ""), + getattr(owner, "tertiary_node_id", ""), serving} + fenced = set(dev.fenced_node_ids or []) + out = set() + for nid in want: + node = nodes_by_id.get(nid) + if not nid or node is None or node.status == _STATUS_REMOVED: + continue + if nid in fenced and nid != serving: + continue + out.add(nid) + return out + + +def reconcile_lun_attachments(cluster_id, path_timeout=LUN_PATH_TIMEOUT_SEC) -> dict: + """Present every array-created LUN of the cluster to the nodes in + ``desired_hosts`` and only to them, and record its path on each. + + Attaches before it detaches, never detaches the serving node, never + attaches a fenced node. A node already recorded in + ``array_attached_node_ids`` costs no array call, so the periodic run is + free while nothing changes. Returns ``{"attached": [...], "detached": + [...], "errors": {device_id: msg}}``; never raises for one LUN.""" + report: dict = {"attached": [], "detached": [], "errors": {}} + db = _db() + cluster = db.get_cluster_by_id(cluster_id) + if not array_configured(cluster): + return report + nodes = db.get_storage_nodes_by_cluster_id(cluster_id) + nodes_by_id = {n.get_id(): n for n in nodes} + work = [] + for owner, dev in _managed_devices(nodes): + if owner.status == _STATUS_REMOVED: + continue + want = desired_hosts(owner, dev, nodes_by_id) + have = set(dev.array_attached_node_ids or []) + missing_paths = [nid for nid in want + if nid != owner.get_id() and nid not in (dev.node_paths or {})] + if want != have or missing_paths: + work.append((owner, dev, want)) + if not work: + return report + try: + plugin = array.plugin_for_cluster(cluster) + except array.ArrayError as e: + for _, dev, _ in work: + report["errors"][dev.get_id()] = str(e) + logger.error("LUN attachments of cluster %s not reconciled: %s", cluster_id, e) + return report + + for owner, dev, want in work: + try: + _reconcile_one(db, plugin, owner, dev, want, nodes_by_id, path_timeout, report) + except Exception as e: # noqa: BLE001 - one LUN never stops the rest + report["errors"][dev.get_id()] = str(e) + logger.error("LUN %s of device %s: attachments not reconciled: %s", + dev.array_lun_id, dev.get_id(), e) + if report["errors"]: + _event(cluster_id, f"Array LUN attachments incomplete: {report['errors']}", error=True) + return report + + +def _reconcile_one(db, plugin, owner, dev, want, nodes_by_id, path_timeout, report): + lun_id = dev.array_lun_id + serving = dev.serving_node() + host_of = {} + for nid in want: + host_of[nid] = ensure_node_host(plugin, nodes_by_id[nid], db) + on_array = set(plugin.get(lun_id).get("hosts") or []) + node_of_host = {} + for n in nodes_by_id.values(): + hid = getattr(n, "array_host_id", "") or "" + if hid: + node_of_host[hid] = n.get_id() + for nid in sorted(want): + if host_of[nid] not in on_array: + plugin.attach(lun_id, host_of[nid]) + report["attached"].append((dev.get_id(), nid)) + logger.info("LUN %s of device %s attached to node %s", lun_id, dev.get_id(), nid) + for host_id in sorted(on_array): + nid = node_of_host.get(host_id) + if nid is None: + # A host no node of the cluster is registered as: left alone + # (release_node_luns detaches a deleted node's host). + logger.warning("LUN %s of device %s is presented to unknown host %s; left alone", + lun_id, dev.get_id(), host_id) + continue + if nid in want or nid == serving: + continue + plugin.detach(lun_id, host_id) + report["detached"].append((dev.get_id(), nid)) + logger.info("LUN %s of device %s detached from node %s", lun_id, dev.get_id(), nid) + + paths = {} + for nid in want: + if nid == owner.get_id() or nid in (dev.node_paths or {}): + continue + path = _lun_path_on(nodes_by_id[nid], dev, path_timeout) + if path: + paths[nid] = {"path": path, "bdev": ""} + else: + report["errors"][dev.get_id()] = f"LUN not visible on node {nid} yet" + attached = sorted(want) + device_id = dev.get_id() + + def _mut(n): + for d in n.nvme_devices or []: + if d.get_id() != device_id: + continue + d.array_attached_node_ids = attached + np = dict(d.node_paths or {}) + # A node that lost the role loses its path; the serving node's + # entry (written by the takeover) is never dropped here. + for nid in list(np): + if nid not in want and nid != d.serving_node() and nid != n.get_id(): + np.pop(nid) + for nid, p in paths.items(): + np.setdefault(nid, p) + d.node_paths = np + return True + return False + db.atomic_update(owner, _mut) + + +_reconcile_threads: dict = {} +_reconcile_lock = threading.Lock() + + +def reconcile_in_background(cluster) -> bool: + """Start ``reconcile_lun_attachments`` for the cluster in a daemon + thread unless one is running (the periodic hook of the health-check + service). Returns whether one was started.""" + if not array_configured(cluster): + return False + cid = cluster.get_id() + with _reconcile_lock: + t = _reconcile_threads.get(cid) + if t is not None and t.is_alive(): + return False + t = threading.Thread(target=_safe_reconcile, args=(cid,), daemon=True) + _reconcile_threads[cid] = t + t.start() + return True + + +def _safe_reconcile(cluster_id): + try: + from simplyblock_core.models.lock import DbLock + with DbLock(f"array_luns/{cluster_id}", timeout=0): + reconcile_lun_attachments(cluster_id) + except Exception as e: # noqa: BLE001 - a background pass + logger.warning("LUN attachment reconcile of cluster %s: %s", cluster_id, e) + + +def reconcile_after_role_change(cluster_id) -> dict: + """The hook after LVS roles were assigned (activation, expansion): one + reconcile pass, synchronous, under the cluster's array lock. Never + raises; problems are in the report and an event.""" + try: + cluster = _db().get_cluster_by_id(cluster_id) + if not array_configured(cluster): + return {"attached": [], "detached": [], "errors": {}} + from simplyblock_core.models.lock import DbLock + with DbLock(f"array_luns/{cluster_id}"): + return reconcile_lun_attachments(cluster_id) + except Exception as e: # noqa: BLE001 + logger.error("LUN attachments of cluster %s not reconciled: %s", cluster_id, e) + return {"attached": [], "detached": [], "errors": {"*": str(e)}} + + +# ------------------------------------------------------------------ growth + +def grow_luns(cluster_id, factor, visible_timeout=LUN_GROW_VISIBLE_TIMEOUT_SEC) -> dict: + """Grow every array-created LUN of the cluster by ``factor`` (> 1) at the + array, then wait until every ONLINE node the LUN is presented to sees the + new size (agent rescan; dm-multipath maps are resized too). + + Returns ``{"luns": {device_id: {"lun_id", "old_size", "new_size"}}, + "errors": {device_id: msg}}``. The device growth itself (alceml, weights) + is the regular grow-devices path that follows. + + Raises: + ValueError: not an array cluster, or a bad factor. + """ + try: + factor = float(factor) + except (TypeError, ValueError) as e: + raise ValueError("the growth factor must be a number") from e + if not math.isfinite(factor) or factor <= 1.0: + raise ValueError("the growth factor must be greater than 1") + db = _db() + cluster = db.get_cluster_by_id(cluster_id) + if not array_configured(cluster): + raise ValueError("LUN growth through the array needs a multi-attach cluster with an array") + try: + plugin = array.plugin_for_cluster(cluster) + except array.ArrayError as e: + raise ValueError(str(e)) from e + nodes = db.get_storage_nodes_by_cluster_id(cluster_id) + nodes_by_id = {n.get_id(): n for n in nodes} + out: dict = {"luns": {}, "errors": {}} + for owner, dev in _managed_devices(nodes): + if owner.status == _STATUS_REMOVED or dev.status in ("removed", "failed", "failed_and_migrated"): + continue + did = dev.get_id() + try: + old = int(plugin.get(dev.array_lun_id).get("size") or 0) or int(dev.size) + target = math.ceil(old * factor) + plugin.resize(dev.array_lun_id, target) + new = int(plugin.get(dev.array_lun_id).get("size") or target) + out["luns"][did] = {"lun_id": dev.array_lun_id, "old_size": old, "new_size": new} + except Exception as e: # noqa: BLE001 + out["errors"][did] = f"array resize of LUN {dev.array_lun_id} failed: {e}" + continue + err = _wait_visible(dev, nodes_by_id, new, visible_timeout) + if err: + out["errors"][did] = err + return out + + +def _wait_visible(dev, nodes_by_id, size, timeout) -> str: + """Rescan the LUN on every ONLINE node it has a path on until each sees + ``size`` bytes (EBS rounds to GiB, so at least ``size``). Returns "" or + what is still missing.""" + pending = {nid: p.get("path") for nid, p in (dev.node_paths or {}).items() + if p.get("path") and nid in nodes_by_id + and nodes_by_id[nid].status == _STATUS_ONLINE} + deadline = time.monotonic() + timeout + last = {} + while pending: + for nid, path in list(pending.items()): + try: + res, err = nodes_by_id[nid].client(timeout=60, retry=1).rescan_block_device(path) + except Exception as e: # noqa: BLE001 + res, err = None, str(e) + seen = int((res or {}).get("size") or 0) + last[nid] = err or f"sees {seen} bytes" + if seen >= size: + pending.pop(nid) + if not pending: + break + if time.monotonic() >= deadline: + return "new size not visible on " + ", ".join(f"{n} ({last.get(n)})" for n in sorted(pending)) + _sleep(POLL_SEC) + return "" + + +# -------------------------------------------------------- removal / delete + +def release_node_luns(node) -> None: + """A storage node is being deleted: detach its own array-created LUNs + from every host, and detach other nodes' LUNs from its host unless it + serves them. Nothing is deleted (only a cluster delete deletes LUNs). + Failures are logged and reported as an event, never raised.""" + try: + db = _db() + cluster = db.get_cluster_by_id(node.cluster_id) + if not array_configured(cluster): + return + nodes = db.get_storage_nodes_by_cluster_id(node.cluster_id) + work = [] + node_host = getattr(node, "array_host_id", "") or "" + for owner, dev in _managed_devices(nodes): + if owner.get_id() == node.get_id(): + work.append((dev, None)) + elif node_host and dev.serving_node() != node.get_id(): + work.append((dev, node_host)) + if not work: + return + plugin = array.plugin_for_cluster(cluster) + except Exception as e: # noqa: BLE001 + logger.error("LUNs of node %s not released: %s", node.get_id(), e) + _event(node.cluster_id, f"LUNs of node {node.get_id()} not released at the array: {e}", error=True) + return + errors = [] + for dev, host in work: + try: + hosts = [host] if host else list(plugin.get(dev.array_lun_id).get("hosts") or []) + for h in hosts: + # The node is gone; its host may not cooperate. + plugin.detach(dev.array_lun_id, h, force=True) + except Exception as e: # noqa: BLE001 + errors.append(f"{dev.array_lun_id}: {e}") + if errors: + logger.error("LUNs of node %s not fully released: %s", node.get_id(), errors) + _event(node.cluster_id, f"LUNs of node {node.get_id()} not fully released: {errors}", error=True) + + +def delete_cluster_luns(cluster, force=False) -> None: + """A cluster is being deleted: delete every LUN its plugin created + (detached from every host first) and the array's KMS secret. + + Raises: + ValueError: a LUN could not be deleted (unless ``force``, which logs + and goes on: the LUNs left are listed in the log). + """ + if not array_configured(cluster): + return + errors = [] + try: + plugin = array.plugin_for_cluster(cluster) + luns = plugin.list_luns() + except Exception as e: # noqa: BLE001 + plugin, luns = None, [] + errors.append(f"array: {e}") + for lun in luns: + try: + plugin.delete_lun(lun["id"]) # type: ignore[union-attr] + logger.info("Cluster %s: LUN %s deleted", cluster.get_id(), lun["id"]) + except Exception as e: # noqa: BLE001 + errors.append(f"{lun['id']}: {e}") + if errors and not force: + raise ValueError(f"array LUNs of the cluster could not be deleted: {'; '.join(errors)}") + if errors: + logger.error("Cluster %s deleted with array LUNs left behind: %s", cluster.get_id(), errors) + if cluster.array_id and cluster.array_has_credentials: + from simplyblock_core import kms + try: + with kms.create_kms_connection(cluster) as conn: + conn.delete_secret(kms.array_secret_path(cluster.get_id(), cluster.array_id)) + except Exception as e: # noqa: BLE001 + if not force: + raise ValueError(f"the array credentials could not be deleted from the KMS: {e}") from e + logger.error("Array credentials of cluster %s left in the KMS: %s", cluster.get_id(), e) diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index 977e7adaf2..30930cbd56 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -2120,6 +2120,10 @@ def _check_resizable(db_controller, dev, snode): raise ValueError("Device runs with a testing or hang bdev under alceml, remove it first") if not dev.alceml_bdev or not dev.nvme_bdev: raise ValueError("Device has no alceml or AIO bdev recorded") + if multi_attach.served_elsewhere(dev, snode.get_id()): + # The alceml runs on the takeover node; this path grows the owner's. + raise ValueError(f"The device's LUN is served by node {dev.serving_node()} after a " + "takeover; grow it once its owner serves it again") if tasks_controller.get_active_dev_restart_task(snode.cluster_id, dev.get_id()): raise ValueError("A restart task is active for the device") @@ -2327,11 +2331,18 @@ def _rebalance_outcome(weights_updated, started, reason): return {"weights_updated": weights_updated, "started": started, "reason": reason} -def grow_cluster_devices(cluster_id, rebalance=True): +def grow_cluster_devices(cluster_id, rebalance=True, lun_factor=None): """Grow every online lblk data device of a cluster to the size of its block device, one device at a time. Devices whose size did not change are left alone; a device that fails does not stop the others. + ``lun_factor`` (multi-attach cluster with an array only): grow every LUN + the array plugin created by this factor at the array first and wait + until every node it is presented to sees the new size + (array_controller.grow_luns); a LUN that fails is reported under + ``errors`` and its device is not grown. The result then carries + ``luns`` too. + When the devices have grown unevenly and ``rebalance`` is set, the weights are updated and the cluster is rebalanced (rebalance_after_growth), unless a device failed to grow. Returns ``{"devices": [...], "errors": @@ -2340,12 +2351,19 @@ def grow_cluster_devices(cluster_id, rebalance=True): ``uniform`` describes the growth before any weight change.""" db_controller = DBController() results, errors = [], {} + lun_report = None + if lun_factor is not None: + from simplyblock_core.controllers import array_controller + lun_report = array_controller.grow_luns(cluster_id, lun_factor) + errors.update(lun_report["errors"]) for node in db_controller.get_storage_nodes_by_cluster_id(cluster_id): if node.is_secondary_node: continue for dev in node.nvme_devices: if dev.status != NVMeDevice.STATUS_ONLINE or dev.bdev_type != "aio": continue + if dev.get_id() in errors: + continue try: results.append(resize_device(dev.get_id())) except ValueError as e: @@ -2360,4 +2378,7 @@ def grow_cluster_devices(cluster_id, rebalance=True): "cluster grow-devices again") else: outcome = rebalance_after_growth(cluster_id) - return {"devices": results, "errors": errors, "uniform": uniform, "rebalance": outcome} + out = {"devices": results, "errors": errors, "uniform": uniform, "rebalance": outcome} + if lun_report is not None: + out["luns"] = lun_report["luns"] + return out diff --git a/simplyblock_core/controllers/events_controller.py b/simplyblock_core/controllers/events_controller.py index 9d1764f8c0..429c86c5ec 100644 --- a/simplyblock_core/controllers/events_controller.py +++ b/simplyblock_core/controllers/events_controller.py @@ -25,6 +25,8 @@ #: Multi-attach array LUN mode: a LUN was (or could not be) taken over by #: another node after its serving node failed. EVENT_LUN_TAKEOVER = "LUN_TAKEOVER" +# Multi-attach array LUN lifecycle (controllers/array_controller). +EVENT_ARRAY_LUN = "ARRAY_LUN" DOMAIN_CLUSTER = "cluster" DOMAIN_MANAGEMENT = "management" diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index 25bb87b231..552eb5d8bd 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -275,10 +275,13 @@ def _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) # ---------------------------------------------------------------- 1. fence def _fence(cluster, dev, failed): - host = multi_attach.array_host(failed) + # The array's own ids when the plugin created the LUN / registered the + # host (array_controller), else the WWN / hostname as in phase 3. + host = multi_attach.array_host_ref(failed) if not host: raise TakeoverError(f"node {failed.get_id()} has no hostname to fence it by") - if not dev.lun_wwn: + lun = multi_attach.array_lun_ref(dev) + if not lun: raise TakeoverError("the device has no LUN WWN to fence") try: plugin = array.plugin_for_cluster(cluster) @@ -289,12 +292,12 @@ def _fence(cluster, dev, failed): except array.ArrayError as e: raise TakeoverError(f"array plugin unavailable: {e}") try: - ok = plugin.fence(dev.lun_wwn, host) + ok = plugin.fence(lun, host) except Exception as e: # noqa: BLE001 - raise TakeoverError(f"array fence of {host} on LUN {dev.lun_wwn} failed: {e}") + raise TakeoverError(f"array fence of {host} on LUN {lun} failed: {e}") if not ok: - raise TakeoverError(f"array did not acknowledge the fence of {host} on LUN {dev.lun_wwn}") - logger.info("LUN %s: node %s (%s) fenced at the array", dev.lun_wwn, failed.get_id(), host) + raise TakeoverError(f"array did not acknowledge the fence of {host} on LUN {lun}") + logger.info("LUN %s: node %s (%s) fenced at the array", lun, failed.get_id(), host) def _record_fenced(db, owner, device_id, node_id): @@ -580,7 +583,8 @@ def unfence_node(device_id, node_id) -> bool: logger.error("Unfence of %s on %s refused: it is the serving node", node_id, device_id) return False try: - if not array.plugin_for_cluster(cluster).unfence(dev.lun_wwn, multi_attach.array_host(node)): + if not array.plugin_for_cluster(cluster).unfence( + multi_attach.array_lun_ref(dev), multi_attach.array_host_ref(node)): logger.error("Unfence of %s on LUN %s not acknowledged", node_id, dev.lun_wwn) return False except Exception as e: # noqa: BLE001 diff --git a/simplyblock_core/kms/__init__.py b/simplyblock_core/kms/__init__.py index b59814c65b..0e3d1cb609 100644 --- a/simplyblock_core/kms/__init__.py +++ b/simplyblock_core/kms/__init__.py @@ -21,6 +21,12 @@ def backup_dek_path(cluster_id: str, backup_id: str) -> str: return f"cluster/{cluster_id}/backup/{backup_id}" +def array_secret_path(cluster_id: str, array_id: str) -> str: + """Where the credentials of a cluster's storage array live in the KMS + (multi-attach array LUN mode, simplyblock_core.array).""" + return f"cluster/{cluster_id}/array/{array_id}" + + def pool_kek_name(pool_id: str) -> str: return f"pool-{pool_id}" diff --git a/simplyblock_core/kms/_base.py b/simplyblock_core/kms/_base.py index a43cfcbaab..1bc6126b9a 100644 --- a/simplyblock_core/kms/_base.py +++ b/simplyblock_core/kms/_base.py @@ -48,3 +48,50 @@ def create_key_encryption_key(self, name: str) -> None: ... @abstractmethod def delete_key_encryption_key(self, name: str) -> None: ... + + # Generic secrets: a flat ``{name: value}`` dict stored at ``path`` (array + # credentials, see simplyblock_core.array). Values go in as ``SecretStr`` + # (plain strings are wrapped) and come back as ``SecretStr``; only the + # persistence boundary of each implementation sees the plaintext. + + @abstractmethod + def put_secret(self, path: str, data: dict) -> None: + """Store ``data`` at ``path``, replacing whatever was there.""" + + @abstractmethod + def get_secret(self, path: str) -> dict: + """The secret stored at ``path``, every value a ``SecretStr``. + + Raises: + KMSException: nothing is stored at ``path``, or the read failed. + """ + + @abstractmethod + def delete_secret(self, path: str) -> None: + """Delete the secret at ``path``; deleting a missing one succeeds.""" + + +def secret_plaintext(data: dict) -> dict: + """``data`` with every value unwrapped to a plain string, for the + persistence boundary of a KMS only. Keys must be strings; values are + ``SecretStr`` or ``str`` (anything else is refused, so a nested structure + cannot carry an unmasked secret past the wrapper).""" + if not isinstance(data, dict): + raise TypeError("a secret is a dict of strings") + out = {} + for k, v in data.items(): + if not isinstance(k, str): + raise TypeError("secret keys must be strings") + if isinstance(v, SecretStr): + out[k] = v.get_secret_value() + elif isinstance(v, str): + out[k] = v + else: + raise TypeError(f"secret value of {k!r} must be a string") + return out + + +def secret_wrapped(data: dict) -> dict: + """``data`` read back from a KMS, every value wrapped in ``SecretStr``.""" + return {str(k): v if isinstance(v, SecretStr) else SecretStr(str(v)) + for k, v in (data or {}).items()} diff --git a/simplyblock_core/kms/_fdb.py b/simplyblock_core/kms/_fdb.py index d86b5a7d17..b7c240f9ff 100644 --- a/simplyblock_core/kms/_fdb.py +++ b/simplyblock_core/kms/_fdb.py @@ -6,7 +6,7 @@ from simplyblock_core.models.cluster import Cluster from simplyblock_core.utils import generate_hex_string -from ._base import KMS +from ._base import KMS, secret_plaintext, secret_wrapped from ._exceptions import KMSException @@ -21,6 +21,12 @@ def __init__(self, cluster: Cluster): def _key(path: str) -> bytes: return f"keys/{path}".encode() + @staticmethod + def _secret_key(path: str) -> bytes: + # A prefix of its own: a generic secret never shares a key with a DEK + # pair stored at the same path. + return f"secrets/{path}".encode() + def create_data_encryption_keys(self, path: str, kek_name: str) -> None: self.import_data_encryption_keys( path, kek_name, @@ -52,3 +58,16 @@ def create_key_encryption_key(self, name: str) -> None: def delete_key_encryption_key(self, name: str) -> None: pass + + def put_secret(self, path: str, data: dict) -> None: + # The persistence boundary, like import_data_encryption_keys. + self._kv_store.set(self._secret_key(path), json.dumps(secret_plaintext(data)).encode()) + + def get_secret(self, path: str) -> dict: + raw = self._kv_store.get(self._secret_key(path)) + if not raw: + raise KMSException(f"No secret found at {path}") + return secret_wrapped(json.loads(raw)) + + def delete_secret(self, path: str) -> None: + self._kv_store.clear(self._secret_key(path)) diff --git a/simplyblock_core/kms/_hcp.py b/simplyblock_core/kms/_hcp.py index a68d56b241..eada641254 100644 --- a/simplyblock_core/kms/_hcp.py +++ b/simplyblock_core/kms/_hcp.py @@ -6,7 +6,7 @@ import hvac.exceptions from pydantic import SecretStr -from ._base import KMS +from ._base import KMS, secret_plaintext, secret_wrapped from ._exceptions import KMSException logger = logging.getLogger(__name__) @@ -137,3 +137,36 @@ def delete_key_encryption_key(self, name: str) -> None: self.client.secrets.transit.delete_key(name=name, mount_point=self.transit_mount) except hvac.exceptions.VaultError as e: raise KMSException("Request failed") from e + + # Generic secrets live in the kv v2 mount under the path itself (DEKs use + # the same mount, under cluster//lvol|backup/...; array credentials + # under cluster//array/, so the two never collide). + + def put_secret(self, path: str, data: dict) -> None: + try: + self.client.secrets.kv.v2.create_or_update_secret( + path=path, secret=secret_plaintext(data), mount_point=self.kv_mount, + ) + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + + def get_secret(self, path: str) -> dict: + try: + data = self.client.secrets.kv.v2.read_secret_version( + path=path, mount_point=self.kv_mount, raise_on_deleted_version=True, + )['data']['data'] + except hvac.exceptions.InvalidPath as e: + raise KMSException(f"No secret found at {path}") from e + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + return secret_wrapped(data) + + def delete_secret(self, path: str) -> None: + try: + self.client.secrets.kv.v2.delete_metadata_and_all_versions( + path=path, mount_point=self.kv_mount, + ) + except hvac.exceptions.InvalidPath: + return + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 2499439572..52686c7f17 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -199,11 +199,21 @@ def is_topology_owned(self) -> bool: # taken over by another node only after its serving node is fenced at the # array, through the plugin named here (entry point in the group # simplyblock.array_plugins, see simplyblock_core.array). Empty = no - # plugin: a takeover is refused (never reopen a LUN unfenced). Credentials - # are resolved from the KMS in a later phase. + # plugin: a takeover is refused (never reopen a LUN unfenced). The array + # is registered through controllers/array_controller (POST + # /clusters/{id}/arrays); its credentials live in the KMS at + # cluster//array/, never here. array_plugin: str = "" array_endpoint: str = "" array_options: dict = default_factory(dict) + # Id of the registered array ("" = none, or configured before arrays were + # registered) and whether credentials were stored for it in the KMS. + array_id: str = "" + array_has_credentials: bool = False + # Size in bytes of the LUN created for each node added to the cluster + # while an array is registered; 0 = node adds create no LUN (the LUNs are + # presented by the operator, as in phase 2/3). + array_lun_size: int = 0 tls: bool = False tls_config: dict = default_factory(dict) is_re_balancing: bool = False diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index 2302187b65..27b8aa1521 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -125,12 +125,21 @@ class NVMeDevice(BaseModel): #: (``node_id``). serving_node_id: str = "" #: node_id -> {"path": by-id path of the LUN on that node, "bdev": the - #: base bdev name there}. Only the owner's entry is filled so far. + #: base bdev name there}. The owner's entry, plus the LVS secondary and + #: tertiary once an array-managed LUN is attached to them (bdev ""). node_paths: dict = default_factory(dict) #: Nodes a LUN takeover fenced from this LUN at the array. Phase 3 never #: unfences on its own: a fenced node's host may still hold queued #: writes. Such a node is no takeover candidate for this LUN. fenced_node_ids: list = default_factory(list) + #: The array's id of the LUN when the cluster's array plugin created it + #: (controllers/array_controller); "" for a LUN the operator presented. + #: Only such LUNs are attached, detached, grown and deleted by simplyblock. + array_lun_id: str = "" + #: Nodes the array presents the LUN to, as last applied by + #: array_controller.reconcile_lun_attachments (owner, LVS secondary and + #: tertiary, serving node). + array_attached_node_ids: list = default_factory(list) def serving_node(self) -> str: """The node whose alceml serves this device now (the owner unless a diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index e8a6f9cc95..42633b91b5 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -234,6 +234,10 @@ class StorageNode(BaseNodeObject): # block (multi_attach.cntlid_range). -1 = not assigned; claimed at # add-node or lazily when a multi-attach device subsystem is created. cntlid_block: int = -1 + # Multi-attach clusters with a registered array: the array's id of this + # node's host (ArrayPlugin.ensure_host; the EC2 instance id on EBS). "" + # = not known; the fence then names the host by its hostname. + array_host_id: str = "" # Sync-replication site of the node (operator-supplied at add-node). # Mandatory on a cluster with sync_replication, empty everywhere else. site: str = "" diff --git a/simplyblock_core/multi_attach.py b/simplyblock_core/multi_attach.py index 36955b2bec..d7d3f07a27 100644 --- a/simplyblock_core/multi_attach.py +++ b/simplyblock_core/multi_attach.py @@ -149,7 +149,12 @@ def mark_lun_devices(cluster, snode, devices, blk_entries) -> None: if entry.get("journal"): continue dev.multi_attach = True - dev.lun_wwn = entry.get("wwn") or dev.serial_number or "" + # A LUN the array plugin created carries the identity the plugin + # reported for it (array_controller.provision_node_lun); it is one of + # the WWN / serial the host reports, so lun_keys matches either way. + dev.lun_wwn = entry.get("lun_wwn") or entry.get("wwn") or dev.serial_number or "" + if entry.get("array_lun_id"): + dev.array_lun_id = entry["array_lun_id"] dev.node_paths = { snode.get_id(): { "path": dev.by_id_path or dev.device_path, @@ -204,6 +209,19 @@ def array_host(node) -> str: return host +def array_host_ref(node) -> str: + """How the array plugin names a node's host: the id ``ensure_host`` + returned (``StorageNode.array_host_id``; the EC2 instance id on EBS), + else the hostname (``array_host``).""" + return getattr(node, "array_host_id", "") or array_host(node) + + +def array_lun_ref(device) -> str: + """How the array plugin names a device's LUN: the id the plugin created + it under (``NVMeDevice.array_lun_id``), else its WWN.""" + return getattr(device, "array_lun_id", "") or getattr(device, "lun_wwn", "") or "" + + def standby_candidates(owner, failed_node_id) -> list[str]: """Node ids that may take over ``owner``'s LUNs, in preference order: the owner's LVS secondary, its tertiary, then the owner itself (when it is diff --git a/simplyblock_core/services/health_check_service.py b/simplyblock_core/services/health_check_service.py index 50a9e46ab5..507e0f74f8 100644 --- a/simplyblock_core/services/health_check_service.py +++ b/simplyblock_core/services/health_check_service.py @@ -11,6 +11,7 @@ utils, ) from simplyblock_core.controllers import ( + array_controller, device_controller, device_events, health_controller, @@ -686,6 +687,13 @@ def main(): continue clusters = db.get_clusters() for cluster in clusters: + # Multi-attach cluster with an array: keep every LUN presented to + # its owner's current LVS secondary / tertiary (roles move on + # node removal and migration). No-op on every other cluster. + try: + array_controller.reconcile_in_background(cluster) + except Exception as e: + logger.error(f"Array LUN reconcile of {cluster.get_id()} not started: {e}") for node in db.get_storage_nodes_by_cluster_id(cluster.get_id()): node_id = node.get_id() if node_id not in threads_maps or threads_maps[node_id].is_alive() is False: diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index cc0b21f1d1..257920920f 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -51,6 +51,7 @@ ) from simplyblock_core.controllers import ( activation_gate, + array_controller, device_controller, device_events, health_controller, @@ -4425,6 +4426,25 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, lblk_configured = node_config.get("lblk_devices") or [] lblk_mode = cluster.device_mode == constants.DEVICE_MODE_LBLK + # Multi-attach cluster with a registered array and a LUN size: the + # node's data LUN is created and presented here (the host's own + # config names the journal device only), then added like any + # configured lblk device. No-op on every other cluster. + array_host_id = "" + if lblk_mode and array_controller.provisions_luns(cluster): + if not lblk_configured: + logger.error( + "This cluster creates each node's LUN on its array, but the node " + "config carries no 'lblk_devices' for the journal; run " + "'sn configure --lblk ' first.") + return False + try: + lun_entry, array_host_id = array_controller.provision_node_lun( + cluster, snode_api, node_info, node_config, lblk_configured) + except array_controller.ArrayLifecycleError as e: + logger.error(f"Could not provision the node's array LUN: {e}") + return False + lblk_configured = array_controller.add_lun_to_config(lblk_configured, lun_entry) if lblk_mode: if not lblk_configured: logger.error( @@ -4856,6 +4876,7 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, # lblk mode: persist the configured selection (with serial identity) # so restart can re-resolve devices without the host config file. snode.lblk_devices = lblk_configured if lblk_mode else [] + snode.array_host_id = array_host_id snode.hostname = hostname snode.k8s_node_name = node_info['hostname'] snode.host_nqn = subsystem_nqn @@ -5350,6 +5371,9 @@ def delete_storage_node(node_id, force=False): if snode.pod_network: # The Service held the node's address; with the record gone nothing names it. _delete_spdk_service(snode, snode.rpc_port, snode.cluster_id) + # Multi-attach cluster with an array: detach the node's LUNs from every + # host and other LUNs from its host (nothing is deleted; no-op otherwise). + array_controller.release_node_luns(snode) snode.remove(db_controller.kv_store) for node in db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id): diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index d0fa05313d..111c676b43 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1641,6 +1641,11 @@ def resolve_lblk_entries(configured_entries, host_devices): resolved_entry["partuuid"] = live.get("partuuid") or entry.get("partuuid", "") if entry.get("journal"): resolved_entry["journal"] = True + # A LUN the cluster's array plugin created (multi-attach): its + # identity at the array, for multi_attach.mark_lun_devices. + for key in ("lun_wwn", "array_lun_id"): + if entry.get(key): + resolved_entry[key] = entry[key] resolved.append(resolved_entry) return resolved, missing diff --git a/simplyblock_web/api/internal/_node_info.py b/simplyblock_web/api/internal/_node_info.py index e82f4ddca2..ef57a7cf43 100644 --- a/simplyblock_web/api/internal/_node_info.py +++ b/simplyblock_web/api/internal/_node_info.py @@ -168,6 +168,10 @@ def _amazon_info() -> dict | None: "cloud": "amazon", "ip": data["privateIp"], "public_ip": "", + # Multi-attach clusters on EBS (simplyblock_core.array.aws_ebs): + # volumes attach only within their zone. + "availability_zone": data.get("availabilityZone", ""), + "region": data.get("region", ""), } except Exception: return None diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index f223b2666f..279337dd08 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -33,6 +33,7 @@ from .._sse import WATCH_RESPONSES, WatchParam, sse_response from .alert import api as alert_api from .arbitration import api as arbitration_api +from .array import api as array_api from .backup import api as backup_api from .consistency_group import api as consistency_group_api from .group_backup import api as group_backup_api @@ -426,22 +427,31 @@ class DeviceGrowthDTO(BaseModel): errors: dict[str, str] uniform: bool rebalance: DeviceRebalanceDTO | None = None + luns: dict[str, dict] | None = None @instance_api.post('/grow-devices', name='clusters:grow_devices', response_model=DeviceGrowthDTO) -def grow_devices(cluster: Cluster, rebalance: bool = True) -> DeviceGrowthDTO: +def grow_devices(cluster: Cluster, rebalance: bool = True, + lun_factor: Annotated[float | None, Query(gt=1)] = None) -> DeviceGrowthDTO: """Grow every online lblk data device of the cluster to the size of its block device, one device at a time. Used after the LUNs of the cluster were grown on the storage array. If they have grown unevenly, the device weights are set to the new sizes and the cluster is rebalanced, unless - ``rebalance`` is false.""" + ``rebalance`` is false. ``lun_factor`` (multi-attach cluster with a + registered array): grow every LUN by this factor at the array first.""" if cluster.device_mode != constants.DEVICE_MODE_LBLK: raise HTTPException(409, 'Online device growth needs a cluster in lblk device mode') - return DeviceGrowthDTO(**device_controller.grow_cluster_devices(cluster.get_id(), rebalance=rebalance)) + try: + extra = {"lun_factor": lun_factor} if lun_factor is not None else {} + result = device_controller.grow_cluster_devices(cluster.get_id(), rebalance=rebalance, **extra) + except ValueError as e: + raise HTTPException(409, str(e)) + return DeviceGrowthDTO(**result) instance_api.include_router(alert_api, prefix='/alerts') instance_api.include_router(arbitration_api, prefix='/arbitration') +instance_api.include_router(array_api, prefix='/arrays') instance_api.include_router(storage_node_api, prefix='/storage-nodes') instance_api.include_router(task_api, prefix='/tasks') instance_api.include_router(pool_api, prefix='/storage-pools') diff --git a/simplyblock_web/api/v2/cluster/array.py b/simplyblock_web/api/v2/cluster/array.py new file mode 100644 index 0000000000..1651cf095c --- /dev/null +++ b/simplyblock_web/api/v2/cluster/array.py @@ -0,0 +1,79 @@ +"""The storage array of a multi-attach cluster (array LUN mode, phase 4). + +simplyblock creates, presents, grows, fences and deletes the cluster's LUNs +through the array plugin registered here. Credentials go to the KMS +(cluster//array/) and are never returned; everything else is +kept on the cluster record. See simplyblock_core.controllers.array_controller. +""" + +from typing import Annotated, Any + +from fastapi import APIRouter, HTTPException, Response +from pydantic import BaseModel, Field, SecretStr + +from simplyblock_core.controllers import array_controller + +from .. import util +from .._dependencies import Cluster + +api = APIRouter() + + +class ArrayParams(BaseModel): + plugin: Annotated[str, Field(min_length=1, description="Plugin name (entry point in simplyblock.array_plugins), e.g. aws-ebs")] + endpoint: str = "" + options: dict[str, Any] = Field(default_factory=dict) + # Stored in the KMS only; masked in every representation of this model. + credentials: dict[str, SecretStr] = Field(default_factory=dict) + lun_size: Annotated[util.Size, Field(description="Size of the LUN created for each added node; 0 = none")] = 0 + + +class ArrayDTO(BaseModel): + id: str + plugin: str + endpoint: str + options: dict[str, Any] + lun_size: int + has_credentials: bool + capabilities: dict[str, Any] | None = None + + +@api.post('/', name='clusters:arrays:create', status_code=201, response_model=ArrayDTO) +def register(cluster: Cluster, parameters: ArrayParams) -> ArrayDTO: + """Register the cluster's storage array (or re-register it, e.g. to + rotate its credentials). The plugin is loaded and connected with the + credentials and its capabilities read before anything is saved.""" + try: + view = array_controller.register_array( + cluster.get_id(), parameters.plugin, parameters.endpoint, parameters.options, + parameters.credentials, parameters.lun_size) + except ValueError as e: + raise HTTPException(422, str(e)) + return ArrayDTO(**view) + + +@api.get('/', name='clusters:arrays:list', response_model=list[ArrayDTO]) +def list_arrays(cluster: Cluster) -> list[ArrayDTO]: + return [ArrayDTO(**a) for a in array_controller.get_arrays(cluster.get_id())] + + +@api.get('/{array_id}', name='clusters:arrays:get', response_model=ArrayDTO) +def get(cluster: Cluster, array_id: str) -> ArrayDTO: + for a in array_controller.get_arrays(cluster.get_id()): + if a["id"] == array_id: + return ArrayDTO(**a) + raise HTTPException(404, f"array {array_id} not found") + + +@api.delete('/{array_id}', name='clusters:arrays:delete', status_code=204, + responses={204: {"content": None}}) +def delete(cluster: Cluster, array_id: str, force: bool = False) -> Response: + """Remove the registration and its credentials. Refused while LUNs + created through the array exist, unless ``force``.""" + try: + array_controller.delete_array(cluster.get_id(), array_id, force=force) + except KeyError as e: + raise HTTPException(404, str(e.args[0] if e.args else e)) + except ValueError as e: + raise HTTPException(409, str(e)) + return Response(status_code=204) diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index 7125d18dbb..1b74c0a789 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -821,7 +821,9 @@ def rescan_block_device(device_path: str) -> dict: SCSI disks (FC, iSCSI, SAS) are rescanned through ``/sys/block//device/rescan``; NVMe namespaces through ``rescan_controller`` of every controller that reaches them, which also - covers the head device of native NVMe multipathing. Other devices, such + covers the head device of native NVMe multipathing. A dm-multipath map + (``dm-N`` with an ``mpath-`` uuid) has each of its path devices + rescanned, then ``multipathd resize map``. Other devices, such as virtio-blk, pick up a resize on their own, so their size is just read. Partitions are refused: growing one is a partition-table change, not a rescan. @@ -843,19 +845,39 @@ def rescan_block_device(device_path: str) -> dict: raise ValueError(f"{name} is a partition, which cannot be resized online") rescanned: list[str] = [] - if _re.match(r"^nvme\d+n\d+$", name): - # A plain namespace links device -> its controller; the head device - # of native multipathing links device -> the subsystem, whose - # controllers are its nvme* children. - candidates = [f"{sys_dev}/device/rescan_controller"] - candidates += sorted(_glob.glob(f"{sys_dev}/device/nvme*/rescan_controller")) - for path in candidates: - if os.path.exists(path): - _write_sysfs(path, "1") - rescanned.append(path) - elif os.path.exists(f"{sys_dev}/device/rescan"): - _write_sysfs(f"{sys_dev}/device/rescan", "1") - rescanned.append(f"{sys_dev}/device/rescan") + + def _rescan_disk(disk: str, sys_disk: str) -> None: + if _re.match(r"^nvme\d+n\d+$", disk): + # A plain namespace links device -> its controller; the head + # device of native multipathing links device -> the subsystem, + # whose controllers are its nvme* children. + candidates = [f"{sys_disk}/device/rescan_controller"] + candidates += sorted(_glob.glob(f"{sys_disk}/device/nvme*/rescan_controller")) + for path in candidates: + if os.path.exists(path): + _write_sysfs(path, "1") + rescanned.append(path) + elif os.path.exists(f"{sys_disk}/device/rescan"): + _write_sysfs(f"{sys_disk}/device/rescan", "1") + rescanned.append(f"{sys_disk}/device/rescan") + + if _re.match(r"^dm-\d+$", name): + # A dm-multipath map (array LUN over several paths): every path + # device is rescanned first, then multipathd resizes the map, which + # only grows once all its paths report the new size. + slaves = sorted(os.listdir(f"{sys_dev}/slaves")) if os.path.isdir(f"{sys_dev}/slaves") else [] + for slave in slaves: + if _re.match(r"^[a-zA-Z0-9_\-]+$", slave): + _rescan_disk(slave, f"/sys/class/block/{slave}") + map_name = _read_sysfs(f"{sys_dev}/dm/name") + uuid = _read_sysfs(f"{sys_dev}/dm/uuid") + if map_name and uuid.startswith("mpath-") and _re.match(r"^[A-Za-z0-9_.\-]+$", map_name): + out, err, rc = shell_utils.run_command(f"multipathd resize map {map_name}") + if rc != 0 or "fail" in (out or "").lower(): + raise ValueError(f"multipathd resize map {map_name} failed: {err or out}") + rescanned.append(f"multipathd:{map_name}") + else: + _rescan_disk(name, sys_dev) sectors = _read_sysfs(f"{sys_dev}/size") if not sectors: diff --git a/tests/unit/test_array_lifecycle.py b/tests/unit/test_array_lifecycle.py new file mode 100644 index 0000000000..4d45947612 --- /dev/null +++ b/tests/unit/test_array_lifecycle.py @@ -0,0 +1,1016 @@ +"""Multi-attach array LUN mode, phase 4: array plugins, credentials in the +KMS, registration, the LUN lifecycle, and the AWS EBS io2 plugin. + + - KMS generic secrets (LocalKMS, HCPClient): stored plaintext only at the + persistence boundary, read back as SecretStr, own key prefix. + - plugin interface: connect with KMS credentials, ownership tags, default + fence = forced detach; FakeArrayPlugin implements everything in memory. + - array_controller: register / get / delete (credentials never on the + cluster record), node-add LUN provisioning, attachment reconcile on role + changes (never detaches the serving node, never attaches a fenced one), + growth by factor, node release, cluster delete. + - lun_takeover fences by the array's own LUN / host ids. + - aws_ebs against botocore's Stubber. + - v2 API and CLI front-ends. +""" + +import json +import os +import tempfile +import unittest +from contextlib import nullcontext +from typing import ClassVar +from unittest.mock import MagicMock, patch + +from pydantic import SecretStr + +from simplyblock_core import array, multi_attach, utils +from simplyblock_core.array import aws_ebs +from simplyblock_core.array.fake import FakeArrayPlugin +from simplyblock_core.controllers import array_controller, device_controller +from simplyblock_core.kms import KMSException, array_secret_path +from simplyblock_core.kms._fdb import LocalKMS +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + +PLUGIN = "fake-p4" +CID = "cluster-4" +GIB = 1024 ** 3 + + +# ------------------------------------------------------------------ fakes + +class FakeKV: + def __init__(self): + self.data = {} + + def set(self, k, v): + self.data[k] = v + + def get(self, k): + return self.data.get(k) + + def clear(self, k): + self.data.pop(k, None) + + +class FakeKMS: + """A KMS holding secrets in memory (as plaintext, like the stores).""" + store: ClassVar[dict] = {} + + def __enter__(self): + return self + + def __exit__(self, *a): + return None + + def put_secret(self, path, data): + from simplyblock_core.kms._base import secret_plaintext + FakeKMS.store[path] = secret_plaintext(data) + + def get_secret(self, path): + from simplyblock_core.kms._base import secret_wrapped + if path not in FakeKMS.store: + raise KMSException(f"No secret found at {path}") + return secret_wrapped(FakeKMS.store[path]) + + def delete_secret(self, path): + FakeKMS.store.pop(path, None) + + +class FakeDB: + def __init__(self, cluster, nodes): + self.cluster = cluster + self.nodes = {n.get_id(): n for n in nodes} + + def get_cluster_by_id(self, _cid): + return self.cluster + + def get_storage_nodes_by_cluster_id(self, _cid): + return list(self.nodes.values()) + + def get_storage_node_by_id(self, nid): + return self.nodes[nid] + + def atomic_update(self, obj, fn): + fresh = self.cluster if isinstance(obj, Cluster) else self.nodes[obj.get_id()] + fn(fresh) + return fresh + + +def _cluster(plugin=PLUGIN, multi=True, lun_size=0): + cl = Cluster() + cl.uuid = CID + cl.multi_attach = multi + cl.max_fault_tolerance = 2 + cl.array_plugin = plugin + cl.array_lun_size = lun_size + cl.status = Cluster.STATUS_ACTIVE + return cl + + +def _node(nid, sec="", ter="", status=StorageNode.STATUS_ONLINE): + n = StorageNode() + n.uuid = nid + n.cluster_id = CID + n.hostname = f"{nid}_5260" + n.rpc_port = 5260 + n.status = status + n.secondary_node_id, n.tertiary_node_id = sec, ter + n.agent = MagicMock() + n.agent.lun_info.return_value = ([{"by_id_path": f"/dev/disk/by-id/lun-{nid}", + "by_id_paths": [], "device_path": "/dev/sdx"}], None) + n.client = lambda *a, _c=n.agent, **k: _c + return n + + +def _lun_device(owner, lun_id, wwn, size=10 * GIB): + d = NVMeDevice() + d.uuid = f"dev-{owner.get_id()}" + d.node_id = owner.get_id() + d.cluster_id = CID + d.status = NVMeDevice.STATUS_ONLINE + d.multi_attach = True + d.array_lun_id = lun_id + d.lun_wwn = wwn + d.size = size + d.node_paths = {owner.get_id(): {"path": f"/dev/disk/by-id/own-{owner.get_id()}", "bdev": "aio"}} + owner.nvme_devices = [d] + return d + + +class _Case(unittest.TestCase): + def setUp(self): + FakeArrayPlugin.reset() + FakeKMS.store = {} + array.register_plugin(PLUGIN, FakeArrayPlugin) + self.addCleanup(array.unregister_plugin, PLUGIN) + for target, attr, kw in ( + (array_controller, "_event", {}), + (array_controller, "_sleep", {}), + ("simplyblock_core.kms.create_kms_connection", None, {"side_effect": lambda c: FakeKMS()}), + ("simplyblock_core.models.lock.DbLock", None, {"side_effect": lambda *a, **k: nullcontext()})): + p = patch(target, **kw) if attr is None else patch.object(target, attr, **kw) + p.start() + self.addCleanup(p.stop) + + def use_db(self, db): + p = patch.object(array_controller, "_db", return_value=db) + p.start() + self.addCleanup(p.stop) + return db + + def plugin(self, cluster): + return array.plugin_for_cluster(cluster) + + +# -------------------------------------------------------------------- KMS + +class TestLocalKmsSecrets(unittest.TestCase): + + def _kms(self): + kv = FakeKV() + with patch("simplyblock_core.kms._fdb.DBController") as dbc: + dbc.return_value.kv_store = kv + return LocalKMS(MagicMock()), kv + + def test_round_trip_plaintext_only_at_rest(self): + kms, kv = self._kms() + kms.put_secret("cluster/c/array/a", {"key": SecretStr("s3cr3t"), "user": "u"}) + raw = kv.data[b"secrets/cluster/c/array/a"] + self.assertEqual(json.loads(raw), {"key": "s3cr3t", "user": "u"}) + got = kms.get_secret("cluster/c/array/a") + self.assertIsInstance(got["key"], SecretStr) + self.assertEqual(got["key"].get_secret_value(), "s3cr3t") + self.assertNotIn("s3cr3t", repr(got)) + + def test_own_prefix_and_delete(self): + kms, kv = self._kms() + kms.import_data_encryption_keys("p", "kek", (SecretStr("aa"), SecretStr("bb"))) + kms.put_secret("p", {"x": "y"}) + self.assertIn(b"keys/p", kv.data) + self.assertIn(b"secrets/p", kv.data) + kms.delete_secret("p") + self.assertNotIn(b"secrets/p", kv.data) + self.assertIn(b"keys/p", kv.data) + kms.delete_secret("p") # idempotent + with self.assertRaises(KMSException): + kms.get_secret("p") + + def test_refuses_non_string_values(self): + kms, _ = self._kms() + with self.assertRaises(TypeError): + kms.put_secret("p", {"x": {"nested": "secret"}}) + + +class TestHcpSecrets(unittest.TestCase): + + def _client(self): + from simplyblock_core.kms._hcp import HCPClient + c = HCPClient.__new__(HCPClient) + c.kv_mount = "kv" + c.client = MagicMock() + return c + + def test_put_get_delete(self): + import hvac.exceptions + c = self._client() + c.put_secret("cluster/c/array/a", {"k": SecretStr("v")}) + c.client.secrets.kv.v2.create_or_update_secret.assert_called_once_with( + path="cluster/c/array/a", secret={"k": "v"}, mount_point="kv") + c.client.secrets.kv.v2.read_secret_version.return_value = {"data": {"data": {"k": "v"}}} + got = c.get_secret("cluster/c/array/a") + self.assertEqual(got["k"].get_secret_value(), "v") + c.client.secrets.kv.v2.read_secret_version.side_effect = hvac.exceptions.InvalidPath() + with self.assertRaises(KMSException): + c.get_secret("x") + c.client.secrets.kv.v2.delete_metadata_and_all_versions.side_effect = hvac.exceptions.InvalidPath() + c.delete_secret("x") # a missing secret is deleted already + c.client.secrets.kv.v2.delete_metadata_and_all_versions.side_effect = hvac.exceptions.Forbidden() + with self.assertRaises(KMSException): + c.delete_secret("x") + + +# ------------------------------------------------------- plugin interface + +class TestPluginInterface(_Case): + + def test_plugin_for_cluster_connects_with_kms_credentials(self): + cl = _cluster() + cl.array_id, cl.array_has_credentials = "arr-1", True + cl.array_endpoint, cl.array_options = "https://array", {"pool": "p"} + FakeKMS.store[array_secret_path(CID, "arr-1")] = {"password": "pw"} + plugin = array.plugin_for_cluster(cl) + endpoint, creds, opts = FakeArrayPlugin.connected[-1] + self.assertEqual((endpoint, opts), ("https://array", {"pool": "p"})) + self.assertEqual(creds["password"].get_secret_value(), "pw") + self.assertEqual(plugin.cluster_id, CID) + self.assertNotIn("pw", repr(plugin)) + + def test_no_kms_read_without_credentials(self): + cl = _cluster() + with patch("simplyblock_core.kms.create_kms_connection") as kms: + array.plugin_for_cluster(cl) + kms.assert_not_called() + + def test_unreadable_credentials_are_an_array_error(self): + cl = _cluster() + cl.array_id, cl.array_has_credentials = "arr-1", True + with self.assertRaisesRegex(array.ArrayError, "KMS"): + array.plugin_for_cluster(cl) + + def test_connect_failure_is_an_array_error(self): + FakeArrayPlugin.fail.add("connect") + with self.assertRaisesRegex(array.ArrayError, "could not connect"): + array.plugin_for_cluster(_cluster()) + + def test_builtin_aws_ebs_without_entry_point(self): + with patch.object(array, "_entry_point", return_value=None): + self.assertIs(array.plugin_class("aws-ebs"), aws_ebs.AwsEbsPlugin) + + def test_default_fence_is_forced_detach(self): + class Minimal(FakeArrayPlugin): + fence = array.ArrayPlugin.fence + unfence = array.ArrayPlugin.unfence + p = Minimal(cluster_id=CID) + lun = p.create_lun("l", GIB, {}) + p.attach(lun["id"], "h1") + self.assertTrue(p.fence(lun["id"], "h1")) + self.assertIn(("detach", lun["id"], "h1", True), FakeArrayPlugin.calls) + self.assertEqual(p.get(lun["id"])["hosts"], []) + self.assertTrue(p.unfence(lun["id"], "h1")) + self.assertEqual(p.get(lun["id"])["hosts"], ["h1"]) + + def test_fake_lifecycle_and_ownership(self): + p = self.plugin(_cluster()) + lun = p.create_lun("node-a", 5 * GIB, {"x": "1"}) + self.assertEqual(p.create_lun("node-a", 5 * GIB, {})["id"], lun["id"]) # idempotent + self.assertEqual(FakeArrayPlugin.luns[lun["id"]]["tags"][array.TAG_CLUSTER_ID], CID) + p.attach(lun["id"], "h1") + p.attach(lun["id"], "h1") + p.attach(lun["wwn"], "h2") # by WWN + self.assertEqual(p.get(lun["id"])["hosts"], ["h1", "h2"]) + p.resize(lun["id"], 8 * GIB) + with self.assertRaises(array.ArrayError): + p.resize(lun["id"], GIB) + p.detach(lun["id"], "h2") + p.detach(lun["id"], "h2") + self.assertEqual([x["id"] for x in p.list_luns()], [lun["id"]]) + p.delete_lun(lun["id"]) + p.delete_lun(lun["id"]) + self.assertEqual(p.list_luns(), []) + + def test_foreign_lun_is_never_touched(self): + p = self.plugin(_cluster()) + FakeArrayPlugin().add_foreign_lun("lun-x", tags={array.TAG_CLUSTER_ID: "other"}) + for op in (lambda: p.attach("lun-x", "h"), lambda: p.detach("lun-x", "h"), + lambda: p.resize("lun-x", GIB * 9), lambda: p.delete_lun("lun-x"), + lambda: p.get("lun-x")): + with self.assertRaises(array.ForeignLunError): + op() + self.assertIn("lun-x", FakeArrayPlugin.luns) + self.assertEqual(p.list_luns(), []) + + +# ------------------------------------------------------------ registration + +class TestRegistration(_Case): + + def test_register_stores_credentials_in_kms_only(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + view = array_controller.register_array( + CID, PLUGIN, "https://array", {"az": "z"}, {"password": "pw"}, lun_size=GIB) + cl = db.cluster + self.assertTrue(cl.array_id) + self.assertEqual((cl.array_plugin, cl.array_endpoint, cl.array_options, cl.array_lun_size), + (PLUGIN, "https://array", {"az": "z"}, GIB)) + self.assertTrue(cl.array_has_credentials) + self.assertEqual(FakeKMS.store[array_secret_path(CID, cl.array_id)], {"password": "pw"}) + self.assertNotIn("pw", json.dumps(cl.to_dict(unwrap_secrets=True), default=str)) + self.assertNotIn("credentials", view) + self.assertNotIn("pw", json.dumps(view)) + self.assertEqual(view["capabilities"]["max_attachments"], 16) + self.assertEqual(array_controller.get_arrays(CID)[0]["id"], cl.array_id) + # The connect saw the credentials wrapped. + self.assertIsInstance(FakeArrayPlugin.connected[-1][1]["password"], SecretStr) + + def test_reregister_keeps_id_and_drops_old_secret(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + array_controller.register_array(CID, PLUGIN, credentials={"password": "pw"}) + aid = db.cluster.array_id + array_controller.register_array(CID, PLUGIN) + self.assertEqual(db.cluster.array_id, aid) + self.assertFalse(db.cluster.array_has_credentials) + self.assertEqual(FakeKMS.store, {}) + + def test_refusals_save_nothing(self): + db = self.use_db(FakeDB(_cluster(plugin="", multi=False), [])) + with self.assertRaisesRegex(ValueError, "multi-attach"): + array_controller.register_array(CID, PLUGIN) + db.cluster.multi_attach = True + FakeArrayPlugin.fail.add("connect") + with self.assertRaisesRegex(ValueError, "connect"): + array_controller.register_array(CID, PLUGIN, credentials={"p": "x"}) + FakeArrayPlugin.fail.clear() + FakeArrayPlugin.max_attachments = 2 # FTT2 needs 3 hosts per LUN + with self.assertRaisesRegex(ValueError, "at most 2"): + array_controller.register_array(CID, PLUGIN) + with self.assertRaisesRegex(ValueError, "no array plugin"): + array_controller.register_array(CID, "missing") + self.assertEqual((db.cluster.array_plugin, db.cluster.array_id, FakeKMS.store), ("", "", {})) + + def test_delete(self): + a = _node("a") + db = self.use_db(FakeDB(_cluster(plugin=""), [a])) + array_controller.register_array(CID, PLUGIN, credentials={"p": "x"}) + aid = db.cluster.array_id + with self.assertRaises(KeyError): + array_controller.delete_array(CID, "other") + _lun_device(a, "lun-1", "w") + with self.assertRaisesRegex(ValueError, "LUN"): + array_controller.delete_array(CID, aid) + array_controller.delete_array(CID, aid, force=True) + self.assertEqual((db.cluster.array_plugin, db.cluster.array_id), ("", "")) + self.assertEqual(FakeKMS.store, {}) + self.assertEqual(array_controller.get_arrays(CID), []) + + +# ------------------------------------------------------------- node add + +class TestProvisioning(_Case): + + def _api(self, appear_after=1): + api = MagicMock() + calls = {"n": 0} + + def inv(): + calls["n"] += 1 + disks = [{"type": "disk", "name": "nvme0n1", "serial": "local", "size": GIB}] + if calls["n"] > appear_after: + lun = next(iter(FakeArrayPlugin.luns.values())) + disks.append({"type": "disk", "name": "nvme3n1", "serial": lun["wwn"], + "wwn": "eui.x", "size": lun["size"], "by_id_path": "/dev/disk/by-id/x", + "numa_node": 0}) + return disks, None + api.get_blockdevices.side_effect = inv + return api + + def test_provision_creates_attaches_and_waits(self): + cl = _cluster(lun_size=20 * GIB) + info = {"hostname": "worker-1", "system_id": "sys", + "cloud_instance": {"cloud": "amazon", "id": "i-0abc"}} + entry, host = array_controller.provision_node_lun( + cl, self._api(), info, {"socket": 0}, [{"serial": "local", "name": "nvme0n1"}]) + lun = next(iter(FakeArrayPlugin.luns.values())) + self.assertEqual(host, "host-worker-1") + self.assertEqual(FakeArrayPlugin.hosts[host]["initiators"]["ec2_instance_id"], "i-0abc") + self.assertEqual(lun["name"], "simplyblock-worker-1-s0") + self.assertEqual(lun["size"], 20 * GIB) + self.assertEqual(lun["hosts"], [host]) + self.assertEqual((entry["serial"], entry["lun_wwn"], entry["array_lun_id"], entry["name"]), + (lun["wwn"], lun["wwn"], lun["id"], "nvme3n1")) + # A retried add reuses the LUN. + entry2, _ = array_controller.provision_node_lun( + cl, self._api(appear_after=0), info, {"socket": 0}, [entry]) + self.assertEqual(entry2["array_lun_id"], lun["id"]) + self.assertEqual(len(FakeArrayPlugin.luns), 1) + + def test_lun_that_never_appears_fails(self): + cl = _cluster(lun_size=GIB) + with self.assertRaisesRegex(array_controller.ArrayLifecycleError, "did not appear"): + array_controller.provision_node_lun(cl, self._api(appear_after=10 ** 6), + {"hostname": "h"}, {}, [], timeout=0) + + def test_array_failure_fails(self): + FakeArrayPlugin.fail.add("create_lun") + with self.assertRaisesRegex(array_controller.ArrayLifecycleError, "array"): + array_controller.provision_node_lun(_cluster(lun_size=GIB), self._api(), + {"hostname": "h"}, {}, []) + + def test_config_entry_and_journal_flag(self): + entry = {"serial": "vol0a", "array_lun_id": "vol-0a", "lun_wwn": "vol0a"} + out = array_controller.add_lun_to_config([{"serial": "local"}], entry) + self.assertEqual(out, [{"serial": "local", "journal": True}, entry]) + out = array_controller.add_lun_to_config(out, entry) # re-add: replaced, not doubled + self.assertEqual(len(out), 2) + two = array_controller.add_lun_to_config([{"serial": "a"}, {"serial": "b"}], entry) + self.assertFalse(any(e.get("journal") for e in two)) + + def test_resolve_and_mark_carry_the_array_identity(self): + inv = [{"name": "nvme3n1", "device_path": "/dev/nvme3n1", "serial": "vol0a", "wwn": "eui.1", + "size": GIB, "type": "disk"}] + resolved, missing = utils.resolve_lblk_entries( + [{"serial": "vol0a", "lun_wwn": "vol0a", "array_lun_id": "vol-0a"}], inv) + self.assertEqual(missing, []) + self.assertEqual((resolved[0]["lun_wwn"], resolved[0]["array_lun_id"]), ("vol0a", "vol-0a")) + node = _node("a") + dev = NVMeDevice() + dev.serial_number = "vol0a" + multi_attach.mark_lun_devices(_cluster(), node, [dev], resolved) + self.assertEqual((dev.lun_wwn, dev.array_lun_id), ("vol0a", "vol-0a")) + self.assertEqual(multi_attach.array_lun_ref(dev), "vol-0a") + + def test_gates(self): + self.assertFalse(array_controller.provisions_luns(_cluster(lun_size=0))) + self.assertFalse(array_controller.provisions_luns(_cluster(plugin="", lun_size=GIB))) + self.assertFalse(array_controller.provisions_luns(_cluster(multi=False, lun_size=GIB))) + self.assertTrue(array_controller.provisions_luns(_cluster(lun_size=GIB))) + + +# ------------------------------------------------------------ reconcile + +class TestReconcile(_Case): + + def _bed(self): + a = _node("a", "b", "c") + b, c, d = _node("b"), _node("c"), _node("d") + db = self.use_db(FakeDB(_cluster(), [a, b, c, d])) + p = self.plugin(db.cluster) + lun = p.create_lun("a", 10 * GIB, {}) + a.array_host_id = p.ensure_host("a", {}) + p.attach(lun["id"], a.array_host_id) + dev = _lun_device(a, lun["id"], lun["wwn"]) + return db, p, a, dev + + def test_attaches_roles_and_records_paths(self): + db, p, a, dev = self._bed() + rep = array_controller.reconcile_lun_attachments(CID) + self.assertEqual(rep["errors"], {}) + self.assertEqual(sorted(p.get(dev.array_lun_id)["hosts"]), ["host-a", "host-b", "host-c"]) + self.assertEqual(sorted(dev.array_attached_node_ids), ["a", "b", "c"]) + self.assertEqual(dev.node_paths["b"]["path"], "/dev/disk/by-id/lun-b") + self.assertIn("a", dev.node_paths) + self.assertEqual(db.nodes["b"].array_host_id, "host-b") + # Steady state: no array call at all. + FakeArrayPlugin.calls.clear() + FakeArrayPlugin.connected.clear() + array_controller.reconcile_lun_attachments(CID) + self.assertEqual((FakeArrayPlugin.calls, FakeArrayPlugin.connected), ([], [])) + + def test_role_move_attaches_first_then_detaches(self): + db, p, a, dev = self._bed() + array_controller.reconcile_lun_attachments(CID) + a.tertiary_node_id = "d" + FakeArrayPlugin.calls.clear() + array_controller.reconcile_lun_attachments(CID) + ops = [c[:3] for c in FakeArrayPlugin.calls if c[0] in ("attach", "detach")] + self.assertEqual(ops, [("attach", dev.array_lun_id, "host-d"), + ("detach", dev.array_lun_id, "host-c")]) + self.assertNotIn("c", dev.node_paths) + self.assertIn("d", dev.node_paths) + + def test_serving_node_is_never_detached_fenced_never_attached(self): + db, p, a, dev = self._bed() + array_controller.reconcile_lun_attachments(CID) + # A takeover moved the LUN to c and fenced b. + a.status = StorageNode.STATUS_OFFLINE + dev.serving_node_id = "c" + dev.fenced_node_ids = ["b"] + p.fence(dev.array_lun_id, "host-b") + a.secondary_node_id, a.tertiary_node_id = "d", "" + array_controller.reconcile_lun_attachments(CID) + hosts = p.get(dev.array_lun_id)["hosts"] + self.assertIn("host-c", hosts) # serving, lost its role: kept + self.assertNotIn("host-b", hosts) # fenced: not re-attached + self.assertIn("host-d", hosts) + self.assertIn("c", dev.node_paths) + + def test_not_visible_yet_is_retried(self): + db, p, a, dev = self._bed() + db.nodes["b"].agent.lun_info.return_value = ([], None) + rep = array_controller.reconcile_lun_attachments(CID, path_timeout=0) + self.assertIn(dev.get_id(), rep["errors"]) + self.assertNotIn("b", dev.node_paths) + db.nodes["b"].agent.lun_info.return_value = ([{"by_id_path": "/dev/disk/by-id/z"}], None) + array_controller.reconcile_lun_attachments(CID) + self.assertEqual(dev.node_paths["b"]["path"], "/dev/disk/by-id/z") + + def test_noop_without_array_and_for_operator_luns(self): + db, p, a, dev = self._bed() + db.cluster.array_plugin = "" + FakeArrayPlugin.calls.clear() + array_controller.reconcile_lun_attachments(CID) + db.cluster.array_plugin = PLUGIN + dev.array_lun_id = "" # presented by the operator: not ours + array_controller.reconcile_lun_attachments(CID) + self.assertEqual(FakeArrayPlugin.calls, []) + + def test_failure_of_one_lun_is_reported(self): + db, p, a, dev = self._bed() + FakeArrayPlugin.fail.add("attach") + rep = array_controller.reconcile_after_role_change(CID) + self.assertIn(dev.get_id(), rep["errors"]) + array_controller._event.assert_called() + + +# --------------------------------------------------------------- growth + +class TestGrowth(_Case): + + def test_grow_by_factor_and_wait_visible(self): + a = _node("a", "b") + b = _node("b") + db = self.use_db(FakeDB(_cluster(), [a, b])) + p = self.plugin(db.cluster) + lun = p.create_lun("a", 10 * GIB, {}) + dev = _lun_device(a, lun["id"], lun["wwn"]) + dev.node_paths["b"] = {"path": "/dev/disk/by-id/lun-b", "bdev": ""} + a.agent.rescan_block_device.return_value = ({"size": 15 * GIB}, None) + b.agent.rescan_block_device.side_effect = [({"size": 10 * GIB}, None), ({"size": 15 * GIB}, None)] + out = array_controller.grow_luns(CID, 1.5) + self.assertEqual(out["errors"], {}) + self.assertEqual(out["luns"][dev.get_id()]["new_size"], 15 * GIB) + self.assertEqual(FakeArrayPlugin.luns[lun["id"]]["size"], 15 * GIB) + self.assertEqual(b.agent.rescan_block_device.call_count, 2) + + def test_refusals_and_errors(self): + a = _node("a") + db = self.use_db(FakeDB(_cluster(), [a])) + for bad in (1, 0.5, "x", float("inf")): + with self.assertRaises(ValueError): + array_controller.grow_luns(CID, bad) + p = self.plugin(db.cluster) + lun = p.create_lun("a", GIB, {}) + dev = _lun_device(a, lun["id"], lun["wwn"]) + a.agent.rescan_block_device.return_value = ({"size": GIB}, None) + out = array_controller.grow_luns(CID, 2, visible_timeout=0) + self.assertIn("not visible", out["errors"][dev.get_id()]) + FakeArrayPlugin.fail.add("resize") + out = array_controller.grow_luns(CID, 2) + self.assertIn("array resize", out["errors"][dev.get_id()]) + db.cluster.array_plugin = "" + with self.assertRaisesRegex(ValueError, "array"): + array_controller.grow_luns(CID, 2) + + def test_grow_devices_skips_devices_whose_lun_failed(self): + db = MagicMock() + node = MagicMock(is_secondary_node=False) + dev = MagicMock(status=NVMeDevice.STATUS_ONLINE, bdev_type="aio") + dev.get_id.return_value = "d1" + node.nvme_devices = [dev] + db.get_storage_nodes_by_cluster_id.return_value = [node] + db.get_cluster_by_id.return_value.reweight_pending = False + with patch.object(device_controller, "DBController", return_value=db), \ + patch.object(array_controller, "grow_luns", + return_value={"luns": {}, "errors": {"d1": "array resize failed"}}), \ + patch.object(device_controller, "resize_device") as rd, \ + patch.object(device_controller, "cluster_growth_uniformity", return_value=(True, 1, 1)): + out = device_controller.grow_cluster_devices(CID, lun_factor=2) + rd.assert_not_called() + self.assertEqual(out["errors"], {"d1": "array resize failed"}) + self.assertEqual(out["luns"], {}) + + def test_served_elsewhere_is_not_resized_on_the_owner(self): + dev = NVMeDevice() + dev.multi_attach, dev.node_id, dev.serving_node_id = True, "a", "b" + dev.status, dev.bdev_type, dev.alceml_bdev, dev.nvme_bdev = NVMeDevice.STATUS_ONLINE, "aio", "al", "nv" + snode = StorageNode() + snode.uuid, snode.status = "a", StorageNode.STATUS_ONLINE + db = MagicMock() + db.get_cluster_by_id.return_value.device_mode = "lblk" + with patch.object(device_controller.constants, "DEVICE_MODE_LBLK", "lblk"): + with self.assertRaisesRegex(ValueError, "served by node b"): + device_controller._check_resizable(db, dev, snode) + + +# --------------------------------------------------------- release / delete + +class TestReleaseAndDelete(_Case): + + def test_release_node_detaches_without_deleting(self): + a, b = _node("a", "b"), _node("b") + db = self.use_db(FakeDB(_cluster(), [a, b])) + p = self.plugin(db.cluster) + la = p.create_lun("a", GIB, {}) + lb = p.create_lun("b", GIB, {}) + a.array_host_id, b.array_host_id = "host-a", "host-b" + for lun in (la, lb): + p.attach(lun["id"], "host-a") + p.attach(lun["id"], "host-b") + _lun_device(a, la["id"], la["wwn"]) + _lun_device(b, lb["id"], lb["wwn"]) + array_controller.release_node_luns(a) + self.assertEqual(p.get(la["id"])["hosts"], []) + self.assertEqual(p.get(lb["id"])["hosts"], ["host-b"]) + self.assertEqual(len(FakeArrayPlugin.luns), 2) + + def test_release_never_detaches_a_lun_the_node_serves(self): + a, b = _node("a"), _node("b") + db = self.use_db(FakeDB(_cluster(), [a, b])) + p = self.plugin(db.cluster) + lb = p.create_lun("b", GIB, {}) + a.array_host_id = "host-a" + p.attach(lb["id"], "host-a") + dev = _lun_device(b, lb["id"], lb["wwn"]) + dev.serving_node_id = "a" + array_controller.release_node_luns(a) + self.assertEqual(p.get(lb["id"])["hosts"], ["host-a"]) + + def test_delete_cluster_deletes_only_its_luns_and_secret(self): + cl = _cluster() + cl.array_id, cl.array_has_credentials = "arr", True + FakeKMS.store[array_secret_path(CID, "arr")] = {"k": "v"} + p = self.plugin(cl) + lun = p.create_lun("a", GIB, {}) + p.attach(lun["id"], "h") + FakeArrayPlugin().add_foreign_lun("lun-foreign", tags={array.TAG_CLUSTER_ID: "other"}) + array_controller.delete_cluster_luns(cl) + self.assertEqual(sorted(FakeArrayPlugin.luns), ["lun-foreign"]) + self.assertEqual(FakeKMS.store, {}) + + def test_delete_cluster_failure(self): + cl = _cluster() + p = self.plugin(cl) + p.create_lun("a", GIB, {}) + FakeArrayPlugin.fail.add("delete_lun") + with self.assertRaisesRegex(ValueError, "could not be deleted"): + array_controller.delete_cluster_luns(cl) + array_controller.delete_cluster_luns(cl, force=True) # logged, goes on + + def test_noop_on_regular_cluster(self): + array_controller.delete_cluster_luns(_cluster(multi=False)) + array_controller.release_node_luns(_node("a")) + self.assertEqual(FakeArrayPlugin.calls, []) + + +# ------------------------------------------------------------ takeover + +class TestTakeoverFenceIds(unittest.TestCase): + + def test_fence_uses_array_ids(self): + from simplyblock_core.controllers import lun_takeover + FakeArrayPlugin.reset() + array.register_plugin(PLUGIN, FakeArrayPlugin) + self.addCleanup(array.unregister_plugin, PLUGIN) + failed = _node("a") + failed.array_host_id = "i-0abc" + dev = NVMeDevice() + dev.lun_wwn, dev.array_lun_id = "vol0abc", "vol-0abc" + lun_takeover._fence(_cluster(), dev, failed) + self.assertEqual(FakeArrayPlugin.calls, [("fence", "vol-0abc", "i-0abc")]) + failed.array_host_id, dev.array_lun_id = "", "" + lun_takeover._fence(_cluster(), dev, failed) + self.assertEqual(FakeArrayPlugin.calls[-1], ("fence", "vol0abc", "a")) + + +# ------------------------------------------------------------ AWS EBS + +class TestAwsEbs(unittest.TestCase): + + def setUp(self): + import boto3 + from botocore.stub import Stubber + self.ec2 = boto3.session.Session( + aws_access_key_id="x", aws_secret_access_key="y", region_name="us-east-1").client("ec2") + self.stub = Stubber(self.ec2) + self.stub.activate() + self.addCleanup(self.stub.deactivate) + self.p = aws_ebs.AwsEbsPlugin(options={"az": "us-east-1a", "poll_interval": 0, "timeout": 5}, + cluster_id=CID) + self.p.ec2 = self.ec2 + self.p._sleep = lambda s: None + + def _vol(self, vid="vol-0abc", state="available", size=10, atts=(), tags=None): + if tags is None: + tags = {array.TAG_CLUSTER_ID: CID} + return {"VolumeId": vid, "State": state, "Size": size, "AvailabilityZone": "us-east-1a", + "Attachments": [{"InstanceId": i, "State": s, "VolumeId": vid} for i, s in atts], + "Tags": [{"Key": k, "Value": v} for k, v in tags.items()]} + + def _describe(self, vol, vid="vol-0abc"): + self.stub.add_response("describe_volumes", {"Volumes": [vol]}, {"VolumeIds": [vid]}) + + def test_ids(self): + self.assertEqual(aws_ebs.volume_wwn("vol-0abc"), "vol0abc") + self.assertEqual(aws_ebs.volume_id_of("vol0abc"), "vol-0abc") + self.assertEqual(aws_ebs.volume_id_of("vol-0abc"), "vol-0abc") + with self.assertRaises(array.ArrayError): + aws_ebs.volume_id_of("0x600a") + + def test_create_new_volume(self): + self.stub.add_response("describe_volumes", {"Volumes": []}) + self.stub.add_response("create_volume", {"VolumeId": "vol-0abc", "State": "creating"}, { + "AvailabilityZone": "us-east-1a", "Size": 20, "VolumeType": "io2", "Iops": 3000, + "MultiAttachEnabled": True, "Encrypted": True, + "TagSpecifications": [{"ResourceType": "volume", "Tags": [ + {"Key": "Name", "Value": "lun-a"}, + {"Key": "role", "Value": "data"}, + {"Key": array.TAG_CLUSTER_ID, "Value": CID}, + {"Key": array.TAG_LUN_NAME, "Value": "lun-a"}]}]}) + self._describe(self._vol(state="creating", size=20)) + self._describe(self._vol(size=20)) + out = self.p.create_lun("lun-a", int(19.5 * GIB), {"role": "data"}) + self.assertEqual(out, {"id": "vol-0abc", "wwn": "vol0abc", "size": 20 * GIB}) + self.stub.assert_no_pending_responses() + + def test_create_finds_existing(self): + self.stub.add_response("describe_volumes", {"Volumes": [self._vol()]}) + self._describe(self._vol()) + self.assertEqual(self.p.create_lun("lun-a", GIB, {})["id"], "vol-0abc") + self.stub.assert_no_pending_responses() + + def test_attach_picks_a_free_device_and_waits(self): + self._describe(self._vol(atts=[("i-other", "attached")])) + self.stub.add_response("describe_instances", {"Reservations": [{"Instances": [{ + "InstanceId": "i-1", "RootDeviceName": "/dev/xvda", + "BlockDeviceMappings": [{"DeviceName": "/dev/sdf"}]}]}]}, {"InstanceIds": ["i-1"]}) + self.stub.add_response("attach_volume", {}, {"VolumeId": "vol-0abc", "InstanceId": "i-1", + "Device": "/dev/sdg"}) + self._describe(self._vol(atts=[("i-other", "attached"), ("i-1", "attaching")])) + self._describe(self._vol(atts=[("i-other", "attached"), ("i-1", "attached")])) + self.p.attach("vol0abc", "i-1") + self.stub.assert_no_pending_responses() + + def test_attach_already_attached_is_a_noop(self): + self._describe(self._vol(atts=[("i-1", "attached")])) + self._describe(self._vol(atts=[("i-1", "attached")])) + self.p.attach("vol-0abc", "i-1") + self.stub.assert_no_pending_responses() + + def test_fence_is_forced_detach_and_waits_until_gone(self): + self._describe(self._vol(atts=[("i-1", "attached"), ("i-2", "attached")])) + self.stub.add_response("detach_volume", {}, {"VolumeId": "vol-0abc", "InstanceId": "i-1", + "Force": True}) + self._describe(self._vol(atts=[("i-1", "detaching"), ("i-2", "attached")])) + self._describe(self._vol(atts=[("i-2", "attached")])) + self.assertTrue(self.p.fence("vol-0abc", "i-1")) + self.stub.assert_no_pending_responses() + + def test_detach_not_attached_is_a_noop(self): + self._describe(self._vol(atts=[("i-2", "attached")])) + self.p.detach("vol-0abc", "i-1") + self.stub.assert_no_pending_responses() + + def test_detach_timeout(self): + self.p.options["timeout"] = 0 + self._describe(self._vol(atts=[("i-1", "attached")])) + self.stub.add_response("detach_volume", {}, {"VolumeId": "vol-0abc", "InstanceId": "i-1", + "Force": False}) + self._describe(self._vol(atts=[("i-1", "busy")])) + with self.assertRaisesRegex(array.ArrayError, "timed out"): + self.p.detach("vol-0abc", "i-1") + + def test_resize_grows_and_waits_for_optimizing(self): + self._describe(self._vol(size=10)) + self.stub.add_response("modify_volume", {}, {"VolumeId": "vol-0abc", "Size": 15}) + self.stub.add_response("describe_volumes_modifications", {"VolumesModifications": [ + {"VolumeId": "vol-0abc", "ModificationState": "modifying"}]}, {"VolumeIds": ["vol-0abc"]}) + self.stub.add_response("describe_volumes_modifications", {"VolumesModifications": [ + {"VolumeId": "vol-0abc", "ModificationState": "optimizing"}]}, {"VolumeIds": ["vol-0abc"]}) + self.p.resize("vol-0abc", 15 * GIB) + self.stub.assert_no_pending_responses() + + def test_resize_never_shrinks(self): + self._describe(self._vol(size=10)) + with self.assertRaisesRegex(array.ArrayError, "shrinking"): + self.p.resize("vol-0abc", 5 * GIB) + + def test_resize_failed_modification(self): + self._describe(self._vol(size=10)) + self.stub.add_response("modify_volume", {}, {"VolumeId": "vol-0abc", "Size": 11}) + self.stub.add_response("describe_volumes_modifications", {"VolumesModifications": [ + {"VolumeId": "vol-0abc", "ModificationState": "failed", "StatusMessage": "nope"}]}) + with self.assertRaisesRegex(array.ArrayError, "failed"): + self.p.resize("vol-0abc", 11 * GIB) + + def test_foreign_volume_is_refused(self): + for tags in ({}, {array.TAG_CLUSTER_ID: "other"}): + self._describe(self._vol(atts=[("i-1", "attached")], tags=tags)) + with self.assertRaises(array.ForeignLunError): + self.p.detach("vol-0abc", "i-1", force=True) + self._describe(self._vol(tags=tags)) + with self.assertRaises(array.ForeignLunError): + self.p.delete_lun("vol-0abc") + self.stub.assert_no_pending_responses() + + def test_get_and_list(self): + self._describe(self._vol(size=8, atts=[("i-1", "attached"), ("i-2", "detached")])) + self.assertEqual(self.p.get("vol-0abc"), + {"id": "vol-0abc", "wwn": "vol0abc", "size": 8 * GIB, "hosts": ["i-1"], + "state": "available"}) + self.stub.add_response("describe_volumes", {"Volumes": [self._vol(), self._vol("vol-1", "deleting")]}, + {"Filters": [{"Name": f"tag:{array.TAG_CLUSTER_ID}", "Values": [CID]}]}) + self.assertEqual([x["id"] for x in self.p.list_luns()], ["vol-0abc"]) + + def test_delete_detaches_then_deletes(self): + self._describe(self._vol(state="in-use", atts=[("i-1", "attached")])) + self._describe(self._vol(state="in-use", atts=[("i-1", "attached")])) + self.stub.add_response("detach_volume", {}, {"VolumeId": "vol-0abc", "InstanceId": "i-1", + "Force": True}) + self._describe(self._vol()) + self._describe(self._vol()) + self.stub.add_response("delete_volume", {}, {"VolumeId": "vol-0abc"}) + self.p.delete_lun("vol-0abc") + self.stub.assert_no_pending_responses() + + def test_delete_missing_is_a_noop(self): + self.stub.add_client_error("describe_volumes", "InvalidVolume.NotFound") + self.p.delete_lun("vol-0abc") + + def test_ensure_host_checks_zone(self): + self.stub.add_response("describe_instances", {"Reservations": [{"Instances": [{ + "InstanceId": "i-1", "Placement": {"AvailabilityZone": "us-east-1a"}}]}]}) + self.assertEqual(self.p.ensure_host("w1", {"ec2_instance_id": "i-1"}), "i-1") + self.stub.add_response("describe_instances", {"Reservations": [{"Instances": [{ + "InstanceId": "i-2", "Placement": {"AvailabilityZone": "us-east-1b"}}]}]}) + with self.assertRaisesRegex(array.ArrayError, "cannot cross zones"): + self.p.ensure_host("w2", {"ec2_instance_id": "i-2"}) + with self.assertRaisesRegex(array.ArrayError, "no EC2 instance id"): + self.p.ensure_host("w3", {"hostname": "w3"}) + + def test_connect(self): + p = aws_ebs.AwsEbsPlugin(cluster_id=CID) + with self.assertRaisesRegex(array.ArrayError, "'az'"): + p.connect("", {}, {}) + with self.assertRaisesRegex(array.ArrayError, "go together"): + p.connect("", {"aws_access_key_id": SecretStr("k")}, {"az": "us-east-1a"}) + with self.assertRaisesRegex(array.ArrayError, "volume_type"): + p.connect("", {}, {"az": "us-east-1a", "volume_type": "gp3"}) + with patch("boto3.session.Session") as sess: + client = sess.return_value.client.return_value + client.describe_availability_zones.return_value = {"AvailabilityZones": [{"ZoneName": "us-east-1a"}]} + p.connect("", {"aws_access_key_id": SecretStr("k"), "aws_secret_access_key": SecretStr("s")}, + {"az": "us-east-1a"}) + sess.assert_called_once_with(aws_access_key_id="k", aws_secret_access_key="s", + aws_session_token=None, region_name="us-east-1") + sess.return_value.client.assert_called_once_with("ec2", endpoint_url=None) + self.assertEqual(p.capabilities()["max_attachments"], 16) + self.assertNotIn("s", repr(p).replace("AwsEbsPlugin", "").replace("cluster", "")) + + def test_instance_role_without_credentials(self): + p = aws_ebs.AwsEbsPlugin(cluster_id=CID) + with patch("boto3.session.Session") as sess: + sess.return_value.client.return_value.describe_availability_zones.return_value = { + "AvailabilityZones": [{}]} + p.connect("https://ec2.vpce", {}, {"az": "eu-west-1b"}) + sess.assert_called_once_with(aws_access_key_id=None, aws_secret_access_key=None, + aws_session_token=None, region_name="eu-west-1") + sess.return_value.client.assert_called_once_with("ec2", endpoint_url="https://ec2.vpce") + + +# ----------------------------------------------------------- front-ends + +class TestApi(unittest.TestCase): + + def _client(self): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from simplyblock_web.api.v2._dependencies import _lookup_cluster + from simplyblock_web.api.v2.cluster import array as array_api + app = FastAPI() + app.include_router(array_api.api, prefix="/clusters/{cluster_id}/arrays") + app.dependency_overrides[_lookup_cluster] = lambda: _cluster() + return TestClient(app) + + def test_post_get_delete(self): + view = {"id": "arr", "plugin": "aws-ebs", "endpoint": "", "options": {"az": "z"}, + "lun_size": GIB, "has_credentials": True, "capabilities": {"max_attachments": 16}} + cid = "00000000-0000-0000-0000-000000000001" + with patch.object(array_controller, "register_array", return_value=view) as reg, \ + patch.object(array_controller, "get_arrays", return_value=[dict(view, capabilities=None)]), \ + patch.object(array_controller, "delete_array") as dele: + c = self._client() + r = c.post(f"/clusters/{cid}/arrays/", json={ + "plugin": "aws-ebs", "options": {"az": "z"}, "lun_size": "1G", + "credentials": {"aws_secret_access_key": "TOPSECRET"}}) + self.assertEqual(r.status_code, 201, r.text) + self.assertNotIn("TOPSECRET", r.text) + args = reg.call_args.args + self.assertIsInstance(args[4]["aws_secret_access_key"], SecretStr) + self.assertEqual(args[5], utils.parse_size("1G")) + r = c.get(f"/clusters/{cid}/arrays/") + self.assertEqual(r.json()[0]["id"], "arr") + self.assertNotIn("credentials", r.json()[0]) + self.assertEqual(c.get(f"/clusters/{cid}/arrays/arr").status_code, 200) + self.assertEqual(c.get(f"/clusters/{cid}/arrays/nope").status_code, 404) + self.assertEqual(c.delete(f"/clusters/{cid}/arrays/arr?force=true").status_code, 204) + dele.assert_called_once_with(_cluster().get_id(), "arr", force=True) + dele.side_effect = ValueError("LUNs exist") + self.assertEqual(c.delete(f"/clusters/{cid}/arrays/arr").status_code, 409) + reg.side_effect = ValueError("bad plugin") + self.assertEqual(c.post(f"/clusters/{cid}/arrays/", json={"plugin": "x"}).status_code, 422) + + +class TestCli(unittest.TestCase): + + def test_set_array_reads_and_wraps_credentials(self): + from simplyblock_cli.clibase import CLIWrapperBase + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + json.dump({"aws_access_key_id": "AK", "aws_secret_access_key": "SK"}, f) + self.addCleanup(os.unlink, f.name) + args = MagicMock(cluster_id=CID, plugin="aws-ebs", endpoint="", options='{"az": "z"}', + credentials_file=f.name, lun_size=GIB, json=True) + with patch.object(array_controller, "register_array", return_value={"id": "arr"}) as reg: + CLIWrapperBase.cluster__set_array(MagicMock(), "set-array", args) + a = reg.call_args.args + self.assertEqual((a[0], a[1], a[3], a[5]), (CID, "aws-ebs", {"az": "z"}, GIB)) + self.assertEqual(a[4]["aws_secret_access_key"].get_secret_value(), "SK") + self.assertNotIn("SK", repr(a[4])) + + def test_bad_credentials_file_does_not_echo_it(self): + from simplyblock_cli.clibase import CLIWrapperBase + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + f.write("SECRET-NOT-JSON") + self.addCleanup(os.unlink, f.name) + args = MagicMock(options=None, credentials_file=f.name) + with self.assertRaises(ValueError) as cm: + CLIWrapperBase.cluster__set_array(MagicMock(), "set-array", args) + self.assertNotIn("SECRET", str(cm.exception)) + + +# ------------------------------------------------------------- agent + +class TestAgent(unittest.TestCase): + + def test_amazon_info_reports_the_zone(self): + from simplyblock_web.api.internal import _node_info + doc = {"instanceId": "i-1", "instanceType": "m5", "privateIp": "10.0.0.1", + "availabilityZone": "us-east-1a", "region": "us-east-1"} + fake = MagicMock() + fake.EC2Metadata.return_value.instance_identity_document = doc + with patch.dict("sys.modules", {"ec2_metadata": fake}): + info = _node_info._amazon_info() + self.assertEqual((info["id"], info["availability_zone"], info["region"]), + ("i-1", "us-east-1a", "us-east-1")) + + def test_rescan_dm_multipath_map(self): + from simplyblock_web import node_utils + root = tempfile.mkdtemp() + self.addCleanup(__import__("shutil").rmtree, root) + + def mk(rel, content=""): + path = os.path.join(root, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as fh: + fh.write(content) + mk("sys/class/block/dm-2/size", "4096") + mk("sys/class/block/dm-2/dm/name", "mpatha") + mk("sys/class/block/dm-2/dm/uuid", "mpath-3600a0") + mk("sys/class/block/dm-2/slaves/sdb/x") + mk("sys/class/block/dm-2/slaves/sdc/x") + mk("sys/class/block/sdb/device/rescan") + mk("sys/class/block/sdc/device/rescan") + + def _map(p): + return os.path.join(root, p.lstrip("/")) if p.startswith("/sys/") else p + real_exists, real_isdir, real_listdir = os.path.exists, os.path.isdir, os.listdir + writes = [] + with patch.object(node_utils.os.path, "realpath", return_value="/dev/dm-2"), \ + patch.object(node_utils.os.path, "exists", side_effect=lambda p: real_exists(_map(p))), \ + patch.object(node_utils.os.path, "isdir", side_effect=lambda p: real_isdir(_map(p))), \ + patch.object(node_utils.os, "listdir", side_effect=lambda p: real_listdir(_map(p))), \ + patch.object(node_utils, "_read_sysfs", + side_effect=lambda p: open(_map(p)).read().strip() if real_exists(_map(p)) else ""), \ + patch.object(node_utils, "_write_sysfs", side_effect=lambda p, v: writes.append(p)), \ + patch.object(node_utils.shell_utils, "run_command", return_value=("ok", "", 0)) as run: + res = node_utils.rescan_block_device("/dev/mapper/mpatha") + self.assertEqual(writes, ["/sys/class/block/sdb/device/rescan", "/sys/class/block/sdc/device/rescan"]) + run.assert_called_once_with("multipathd resize map mpatha") + self.assertEqual(res["size"], 4096 * 512) + self.assertIn("multipathd:mpatha", res["rescanned"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/web/api/v2/test_device_endpoints.py b/tests/unit/web/api/v2/test_device_endpoints.py index b42cdef191..a19e913ee8 100644 --- a/tests/unit/web/api/v2/test_device_endpoints.py +++ b/tests/unit/web/api/v2/test_device_endpoints.py @@ -420,6 +420,23 @@ def test_grows_devices_of_lblk_cluster(self, client, db, cluster, monkeypatch): assert response.json()['rebalance'] is None mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID, rebalance=True) + def test_lun_factor_is_passed_through(self, client, db, cluster, monkeypatch): + from unittest.mock import MagicMock + + import simplyblock_web.api.v2.cluster as cluster_module + mock = MagicMock() + mock.grow_cluster_devices.return_value = { + 'devices': [], 'errors': {}, 'uniform': True, + 'luns': {'d1': {'lun_id': 'vol-1', 'old_size': 1, 'new_size': 2}}} + monkeypatch.setattr(cluster_module, 'device_controller', mock) + cluster.device_mode = 'lblk' + + response = client.post(f'{self.URL}?lun_factor=2') + assert response.status_code == 200 + assert response.json()['luns']['d1']['lun_id'] == 'vol-1' + mock.grow_cluster_devices.assert_called_once_with(CLUSTER_ID, rebalance=True, lun_factor=2.0) + assert client.post(f'{self.URL}?lun_factor=1').status_code == 422 + def test_uneven_growth_reports_rebalance(self, client, db, cluster, monkeypatch): from unittest.mock import MagicMock From 79930558624090628c7989820813c32e0857b1d9 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 13:02:45 +0100 Subject: [PATCH 10/50] fix(multi-attach): mypy findings in phases 2-4 The CI type checker failed on 800f763da (claim_cntlid_block assigned an Optional to an int); phases 3-4 added six more: the Python < 3.10 entry-point fallback (requires-python is >= 3.11), getattr with a str default on a typed field, a loop variable reused outside its except block, an untyped work list, and two Optional list assignments in _connect_to_remote_devs. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/array/__init__.py | 6 +----- simplyblock_core/controllers/array_controller.py | 12 ++++++------ simplyblock_core/db_controller.py | 5 +++-- simplyblock_core/storage_node_ops.py | 4 ++-- tests/unit/test_lun_takeover.py | 8 +++----- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/simplyblock_core/array/__init__.py b/simplyblock_core/array/__init__.py index 9077b90974..3ff129b19e 100644 --- a/simplyblock_core/array/__init__.py +++ b/simplyblock_core/array/__init__.py @@ -211,11 +211,7 @@ def unregister_plugin(name: str) -> None: def _entry_point(name: str): - eps = metadata.entry_points() - if hasattr(eps, "select"): - found = list(eps.select(group=ENTRY_POINT_GROUP, name=name)) - else: # pragma: no cover - Python < 3.10 - found = [ep for ep in eps.get(ENTRY_POINT_GROUP, []) if ep.name == name] + found = list(metadata.entry_points(group=ENTRY_POINT_GROUP, name=name)) return found[0] if found else None diff --git a/simplyblock_core/controllers/array_controller.py b/simplyblock_core/controllers/array_controller.py index 5734228855..7b85d00ebd 100644 --- a/simplyblock_core/controllers/array_controller.py +++ b/simplyblock_core/controllers/array_controller.py @@ -108,7 +108,7 @@ def _wrap_credentials(credentials) -> dict: def _managed_devices(nodes): """``(owner, device)`` of every LUN the array plugin created.""" return [(n, d) for n in nodes for d in (n.nvme_devices or []) - if multi_attach.is_multi_attach(d) and getattr(d, "array_lun_id", "")] + if multi_attach.is_multi_attach(d) and d.array_lun_id] def array_view(cluster, capabilities=None) -> dict: @@ -393,12 +393,12 @@ def _lun_path_on(node, dev, timeout): except Exception as e: # noqa: BLE001 logger.warning("lun_info on %s: %s", node.get_id(), e) entries = None - for e in entries or []: - if owner_path and (owner_path in (e.get("by_id_paths") or []) or owner_path == e.get("by_id_path")): + for entry in entries or []: + if owner_path and (owner_path in (entry.get("by_id_paths") or []) or owner_path == entry.get("by_id_path")): return owner_path if entries: - e = entries[0] - return e.get("by_id_path") or e.get("device_path") or "" + first = entries[0] + return first.get("by_id_path") or first.get("device_path") or "" if time.monotonic() >= deadline: return "" _sleep(POLL_SEC) @@ -669,7 +669,7 @@ def release_node_luns(node) -> None: if not array_configured(cluster): return nodes = db.get_storage_nodes_by_cluster_id(node.cluster_id) - work = [] + work: list[tuple] = [] node_host = getattr(node, "array_host_id", "") or "" for owner, dev in _managed_devices(nodes): if owner.get_id() == node.get_id(): diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index 0fdb85bec4..84ce2ec457 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -1777,11 +1777,12 @@ def _claim_cntlid_block_tx(self, tr, node_key, alloc_key, node_id, live_node_ids # node keeps its block (its record still exists). alloc = {nid: b for nid, b in alloc.items() if nid in live_node_ids} used = {int(b) for b in alloc.values()} - block = next((i for i in range(max_blocks) if i not in used), None) - if block is None: + free = next((i for i in range(max_blocks) if i not in used), None) + if free is None: raise ValueError( f"no free controller-ID block: a multi-attach cluster supports " f"at most {max_blocks} storage nodes") + block = free alloc[node_id] = block tr.set(alloc_key, json.dumps(alloc, sort_keys=True).encode()) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 257920920f..278d67bd6a 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2575,7 +2575,7 @@ def _connect_to_remote_devs( continue node_devices = own_loopback elif node.status not in allowed_node_statuses: - node_devices = taken_over.get(node.get_id()) + node_devices = taken_over.get(node.get_id()) or [] if not node_devices: continue else: @@ -2694,7 +2694,7 @@ def _find_remote_bdev(dev): continue node_devices = own_loopback elif node.status not in allowed_node_statuses: - node_devices = taken_over.get(node.get_id()) + node_devices = taken_over.get(node.get_id()) or [] if not node_devices: continue else: diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index 3241a187b8..1ae3b837a0 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -463,15 +463,13 @@ def test_fake_plugin_is_not_registered_by_default(self): def test_entry_point_loading(self): ep = MagicMock(value="pkg:Cls") ep.load.return_value = FakeArrayPlugin - eps = MagicMock() - eps.select.return_value = [ep] - with patch.object(array.metadata, "entry_points", return_value=eps): + with patch.object(array.metadata, "entry_points", return_value=[ep]) as eps: plugin = array.load_plugin("vendor", "https://array", {"pool": "p"}) - eps.select.assert_called_once_with(group=array.ENTRY_POINT_GROUP, name="vendor") + eps.assert_called_once_with(group=array.ENTRY_POINT_GROUP, name="vendor") self.assertIsInstance(plugin, FakeArrayPlugin) self.assertEqual((plugin.endpoint, plugin.options), ("https://array", {"pool": "p"})) ep.load.return_value = object - with patch.object(array.metadata, "entry_points", return_value=eps): + with patch.object(array.metadata, "entry_points", return_value=[ep]): with self.assertRaises(array.ArrayError): array.load_plugin("vendor") From d0de91b1c6442ed041d4221b38df211c1244d84a Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 13:06:26 +0100 Subject: [PATCH 11/50] test(floors): integration fixtures satisfy the resource floors These fixtures predate the resource floors (simplyblock_core.floors); give them enough resources instead of weakening the floors: - dual_ft_e2e: 3 data devices per primary (6 >= n+2k = 5 for 1+2) - dhchap_pool_level: the second node gets one data device (>= 1 per node) - shutdown_no_suspension: a third bystander node, so the cluster is not a two-node one and the shutdown floor guard does not refuse - cluster_suspend_recovery: 5 ONLINE nodes in the ACTIVE cases so the monitor's floor DEGRADED overlay does not apply Co-Authored-By: Claude Opus 5.5 --- .../test_cluster_suspend_recovery.py | 27 ++++++++++++------- tests/integration/test_dhchap_pool_level.py | 3 ++- tests/integration/test_dual_ft_e2e.py | 5 ++-- .../test_shutdown_no_suspension.py | 10 +++++-- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_cluster_suspend_recovery.py b/tests/integration/test_cluster_suspend_recovery.py index e235f29e72..1dc2be8ef1 100644 --- a/tests/integration/test_cluster_suspend_recovery.py +++ b/tests/integration/test_cluster_suspend_recovery.py @@ -143,8 +143,9 @@ def _run(self, nodes, cluster): # ---- baseline behavior preserved ------------------------------------ def test_all_online_active(self): - # Healthy 1x2 cluster: 4 online nodes, devices all online. - nodes = [_node(f"n{i}") for i in range(4)] + # Healthy 1x2 cluster: 5 online nodes (the n+2k node floor), devices + # all online. + nodes = [_node(f"n{i}") for i in range(5)] c = _cluster(distr_ndcs=1, distr_npcs=2) self.assertEqual(self._run(nodes, c), Cluster.STATUS_ACTIVE) @@ -217,14 +218,14 @@ def test_unreachable_node_counts_as_affected(self): def test_unreachable_without_data_plane_loss_not_counted(self): # The gate itself: mgmt-unreachable with the data plane still serving # (quorum says connected) must NOT suspend — an API blip or mgmt-NIC - # flap leaves clients unaffected. + # flap leaves clients unaffected. 5 ONLINE nodes keep the cluster at + # the resource floors (n+2k), so the verdict is the suspend rules'. from simplyblock_core.services import storage_node_monitor as mod nodes = [ _node("n0", status=StorageNode.STATUS_UNREACHABLE), _node("n1", status=StorageNode.STATUS_UNREACHABLE), _node("n2", status=StorageNode.STATUS_UNREACHABLE), - _node("n3", status=StorageNode.STATUS_ONLINE), - ] + ] + [_node(f"on{i}", status=StorageNode.STATUS_ONLINE) for i in range(5)] c = _cluster(distr_ndcs=1, distr_npcs=2) with patch.object(mod, "is_node_data_plane_disconnected_quorum", return_value=False): @@ -254,9 +255,11 @@ def test_removed_node_does_not_count_as_affected(self): _node("n1", status=StorageNode.STATUS_ONLINE), _node("n2", status=StorageNode.STATUS_ONLINE), _node("n3", status=StorageNode.STATUS_ONLINE), + _node("n4", status=StorageNode.STATUS_ONLINE), + _node("n5", status=StorageNode.STATUS_ONLINE), ] c = _cluster(distr_ndcs=1, distr_npcs=2) - # 0 affected, 3 online -> ACTIVE + # 0 affected, 5 online (the n+2k node floor) -> ACTIVE self.assertEqual(self._run(nodes, c), Cluster.STATUS_ACTIVE) @@ -303,20 +306,26 @@ def test_lone_down_stays_active(self): _node("on-1", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.2"), _node("on-2", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.3"), _node("on-3", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.4"), + _node("on-4", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.5"), + _node("on-5", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.6"), ] + # 5 ONLINE besides the DOWN one: at the n+2k node floor. c = _cluster(distr_ndcs=1, distr_npcs=2) self.assertEqual(self._run(nodes, c), Cluster.STATUS_ACTIVE) def test_many_down_nodes_never_suspend(self): # Even several DOWN nodes never tip the cluster to SUSPENDED — they - # recover via port-unblock. 3 DOWN + 1 ONLINE on a 1x2 cluster -> ACTIVE. + # recover via port-unblock. 3 DOWN (> k) + 5 ONLINE (the n+2k node + # floor, so the floors do not degrade it) on a 1x2 cluster -> ACTIVE. nodes = [ _node(f"down-{i}", status=StorageNode.STATUS_DOWN, mgmt_ip=f"10.0.0.{i + 1}", n_online_devs=2, n_offline_devs=0) for i in range(3) ] - nodes.append( - _node("on-1", status=StorageNode.STATUS_ONLINE, mgmt_ip="10.0.0.9")) + nodes += [ + _node(f"on-{i}", status=StorageNode.STATUS_ONLINE, mgmt_ip=f"10.0.0.{i + 10}") + for i in range(5) + ] c = _cluster(distr_ndcs=1, distr_npcs=2) self.assertEqual(self._run(nodes, c), Cluster.STATUS_ACTIVE) diff --git a/tests/integration/test_dhchap_pool_level.py b/tests/integration/test_dhchap_pool_level.py index de022f12b2..9dc6a20183 100644 --- a/tests/integration/test_dhchap_pool_level.py +++ b/tests/integration/test_dhchap_pool_level.py @@ -546,7 +546,8 @@ class TestLvolInheritsDhchapFromPool: def test_lvol_allowed_hosts_set_from_pool(self, db, cluster): """When pool.dhchap=True, add_lvol_ha populates lvol.allowed_hosts from pool.""" node = _write_node(db) - secondary = _write_node(db, uuid="node-dhchap-2", devices=0) + # One data device: every online primary member needs >= 1 (floors). + secondary = _write_node(db, uuid="node-dhchap-2", devices=1) node.secondary_node_id = secondary.get_id() node.write_to_db(db.kv_store) pool = _write_pool(db, hosts=[HOST_A, HOST_B]) diff --git a/tests/integration/test_dual_ft_e2e.py b/tests/integration/test_dual_ft_e2e.py index 2d129866fc..d72a0b8cd8 100644 --- a/tests/integration/test_dual_ft_e2e.py +++ b/tests/integration/test_dual_ft_e2e.py @@ -672,9 +672,10 @@ def cluster_env(ensure_cluster, mock_rpc_servers): n.active_tcp = True n.active_rdma = False n.data_nics = [_make_nic("127.0.0.1")] - # Give each primary node 2 devices (need ndcs+npcs+1=4 total across cluster) + # Give each primary node 3 devices: 6 in all, at or above the 1+2 + # scheme's data-device floor (n+2k = 5, simplyblock_core.floors). devs = [] - for d in range(2): + for d in range(3): dev = NVMeDevice() dev.uuid = str(_uuid_mod.uuid4()) dev.cluster_id = cluster.uuid diff --git a/tests/integration/test_shutdown_no_suspension.py b/tests/integration/test_shutdown_no_suspension.py index c1f3eaef75..001c94c154 100644 --- a/tests/integration/test_shutdown_no_suspension.py +++ b/tests/integration/test_shutdown_no_suspension.py @@ -343,6 +343,12 @@ def bdev_nvme_detach_controller(self, name): detach_calls.append(name) return True peer.rpc_client = lambda **kwargs: _PeerRPC() + # A third node holding no controllers to the dying node: without it the + # cluster is a two-node one and the shutdown floor guard refuses + # (nodes online 1 < floor 2, simplyblock_core.floors). + bystander = _make_node("bystander", status=StorageNode.STATUS_ONLINE, + mgmt_ip="10.0.0.3") + bystander.rpc_client = lambda **kwargs: _PeerRPC() # dying-node SPDK kill + bind sink kill_calls = [] @@ -362,7 +368,7 @@ def bind_device_to_nvme(self, pci): def _get_node(nid): return snode if nid == snode.get_id() else peer db.get_storage_node_by_id.side_effect = _get_node - db.get_storage_nodes_by_cluster_id.return_value = [snode, peer] + db.get_storage_nodes_by_cluster_id.return_value = [snode, peer, bystander] db.get_job_tasks.return_value = [] db.kv_store = MagicMock() @@ -401,7 +407,7 @@ def _set_status(nid, status, caused_by="monitor"): p.start() self.addCleanup(p.stop) fw_mock = MagicMock() # MUST NOT be called: shutdown fences nothing - route_fences([snode, peer], fw_mock) + route_fences([snode, peer, bystander], fw_mock) return { "snode": snode, "peer": peer, "db": db, From f72652c4bf6d3db467d40d8a14be034fd16c32e2 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 15:43:09 +0100 Subject: [PATCH 12/50] feat(multi-attach): sn configure --lblk-journal-only On a multi-attach cluster whose array plugin creates each node's data LUN at add-node, a storage node has only its journal disk at configure time. The existing lblk layout either refuses one device (two-unit minimum) or splits a single whole disk into a journal and a local data partition, and a local data device on a 1+0 multi-attach cluster is not shared: losing the node loses its data. --lblk-journal-only (CLI sn configure and the k8s node_configure) takes exactly one whole disk, named with --blk-names or --blk-serials, flags it as the journal and never partitions it. add-node then adds the array LUN as the data device, which satisfies the two-unit minimum there. Co-Authored-By: Claude Opus 5.5 --- simplyblock_cli/cli-reference.yaml | 5 +++ simplyblock_cli/cli.py | 1 + simplyblock_cli/clibase.py | 10 +++++ simplyblock_core/utils/__init__.py | 30 +++++++++++--- simplyblock_web/node_configure.py | 17 ++++++++ tests/unit/test_lblk_journal_only.py | 60 ++++++++++++++++++++++++++++ 6 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 tests/unit/test_lblk_journal_only.py diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 328c722c30..9a034aed49 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -107,6 +107,11 @@ commands: required: false type: str default: "" + - name: "--lblk-journal-only" + help: "With --lblk on a multi-attach cluster whose array plugin creates each node's data LUN at add-node: the one selected whole disk is the node's journal and nothing else. It is never partitioned, and the node has no local data device; its data device is the array LUN. Needs exactly one disk, named with --blk-names or --blk-serials." + dest: lblk_journal_only + type: bool + action: store_true - name: "--lblk-journal-device" help: "With --lblk and whole disks: dedicate each node's smallest disk to the journal instead of partitioning it into a journal partition and a data partition (the default). Needs at least two disks per node. Partition selections always carry their journal in a split partition." dest: lblk_journal_device diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index c118ca50f9..13f91dcbf7 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -111,6 +111,7 @@ def init_storage_node__configure(self, subparser): subcommand.add_argument('--blk-names', help='Comma separated list of block devices to use (requires --lblk). Each entry is a persistent /dev/disk name, such as /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0 or /dev/disk/by-partuuid/28427de0-1916-4c05-895f-0829cd8790ba. A kernel name (sdb) or kernel path (/dev/sdb) is refused: it states a position in this boot\'s enumeration order, and it would name another disk after a reboot that probes the controllers in another order. The path is taken as given: the node records it, and add-node and every restart open the device by that path alone, never by serial or kernel name. A path that no longer exists is a missing device. Use --blk-serials for a device udev published no persistent name for. Requested devices must be eligible; a busy device is an error.', type=str, default='', dest='blk_names', required=False) subcommand.add_argument('--blk-names-exclude', help='Comma separated list of block devices to exclude (requires --lblk), spelled the way --blk-names takes them. All other eligible disks are used.', type=str, default='', dest='blk_names_exclude', required=False) subcommand.add_argument('--blk-serials', help='Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk).', type=str, default='', dest='blk_serials', required=False) + subcommand.add_argument('--lblk-journal-only', help='With --lblk on a multi-attach cluster whose array plugin creates each node\'s data LUN at add-node: the one selected whole disk is the node\'s journal and nothing else. It is never partitioned, and the node has no local data device; its data device is the array LUN. Needs exactly one disk, named with --blk-names or --blk-serials.', dest='lblk_journal_only', action='store_true') subcommand.add_argument('--lblk-journal-device', help='With --lblk and whole disks: dedicate each node\'s smallest disk to the journal instead of partitioning it into a journal partition and a data partition (the default). Needs at least two disks per node. Partition selections always carry their journal in a split partition.', dest='lblk_journal_device', action='store_true') subcommand.add_argument('--jm-percent', help='Journal size in percent of the node\'s total selected capacity when the journal is carved from a selected partition or disk (requires --lblk, not with --lblk-journal-device). Default: `3`.', type=int, default=3, dest='jm_percent', required=False) subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format.', dest='force', action='store_true') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index b8d6694906..71f1e86f4d 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -256,6 +256,15 @@ def storage_node__configure(self, sub_command, args): lblk_journal_device = getattr(args, 'lblk_journal_device', False) if lblk_journal_device and not lblk: self.parser.error("--lblk-journal-device requires --lblk") + lblk_journal_only = getattr(args, 'lblk_journal_only', False) + if lblk_journal_only: + if not lblk: + self.parser.error("--lblk-journal-only requires --lblk") + if lblk_journal_device: + self.parser.error("--lblk-journal-only and --lblk-journal-device exclude each other") + if not (blk_names or blk_serials): + self.parser.error("--lblk-journal-only needs the journal disk named with " + "--blk-names or --blk-serials") lblk_selection = None if use_lblk: if not lblk: @@ -269,6 +278,7 @@ def storage_node__configure(self, sub_command, args): "names_exclude": [str(x) for x in blk_names_exclude.split(',')] if blk_names_exclude else None, "serials": [str(x) for x in blk_serials.split(',')] if blk_serials else None, "journal_device": bool(lblk_journal_device), + "journal_only": bool(lblk_journal_only), } # The core split is decided by the cluster's vcpu-count when the node # is added or restarted; here the default heuristic lays out a usable diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index f366557505..894d0572c0 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -2714,6 +2714,22 @@ def previous_lblk_dedicated_journal_serials(config_paths=None) -> set: return serials +def lblk_journal_only_layout(lblk_entries, socket=0): + """``sn configure --lblk-journal-only``: the node's one selected whole + disk becomes its journal, flagged, and is never partitioned. The node + then has no local data device: on a multi-attach cluster whose array + plugin creates each node's data LUN at add-node, that LUN is its data + device. Raises ValueError unless exactly one whole disk is selected.""" + if len(lblk_entries) != 1: + raise ValueError( + f"--lblk-journal-only needs exactly one disk for the node on socket {socket}; " + f"selected: {sorted(lblk_entries)}") + name, entry = next(iter(lblk_entries.items())) + if entry.get("type") == "part": + raise ValueError(f"--lblk-journal-only needs a whole disk, {name} is a partition") + return {name: dict(entry, journal=True)} + + def lblk_node_journal_layout(lblk_entries, jm_percent=3, journal_device=False, keep_serials=()): """Give one storage node's devices their journal. @@ -3340,11 +3356,15 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a entry.pop("_pair", None) node_entries[dev_name] = entry try: - node_entries = lblk_node_journal_layout( - node_entries, jm_percent=jm_percent, - journal_device=bool(lblk_selection.get("journal_device")), - keep_serials=lblk_keep_serials) - if node_entries and len(node_entries) < constants.LBLK_MIN_DEVICES_PER_NODE: + if lblk_selection.get("journal_only"): + node_entries = lblk_journal_only_layout(node_entries, nid) + else: + node_entries = lblk_node_journal_layout( + node_entries, jm_percent=jm_percent, + journal_device=bool(lblk_selection.get("journal_device")), + keep_serials=lblk_keep_serials) + if (node_entries and not lblk_selection.get("journal_only") + and len(node_entries) < constants.LBLK_MIN_DEVICES_PER_NODE): raise ValueError( f"lblk mode requires at least {constants.LBLK_MIN_DEVICES_PER_NODE} " f"partitions or SSDs per node; only {len(node_entries)} unit(s) " diff --git a/simplyblock_web/node_configure.py b/simplyblock_web/node_configure.py index a3b39000b0..5ff8bea869 100755 --- a/simplyblock_web/node_configure.py +++ b/simplyblock_web/node_configure.py @@ -220,6 +220,12 @@ def parse_arguments() -> argparse.Namespace: dest='blk_serials', required=False ) + parser.add_argument( + '--lblk-journal-only', + help="With --lblk on a multi-attach cluster whose array plugin creates each node's data LUN at add-node: the one selected whole disk is the node's journal and nothing else. It is never partitioned, and the node has no local data device; its data device is the array LUN. Needs exactly one disk, named with --blk-names or --blk-serials.", + action='store_true', + dest='lblk_journal_only', + ) parser.add_argument( '--lblk-journal-device', help='With --lblk and whole disks: dedicate each node\'s smallest disk to the ' @@ -290,6 +296,16 @@ def validate_arguments(args: argparse.Namespace) -> None: None, "--blk-names/--blk-names-exclude/--blk-serials require --lblk") if getattr(args, 'lblk_journal_device', False) and not lblk: raise argparse.ArgumentError(None, "--lblk-journal-device requires --lblk") + if getattr(args, 'lblk_journal_only', False): + if not lblk: + raise argparse.ArgumentError(None, "--lblk-journal-only requires --lblk") + if getattr(args, 'lblk_journal_device', False): + raise argparse.ArgumentError( + None, "--lblk-journal-only and --lblk-journal-device exclude each other") + if not (args.blk_names or args.blk_serials): + raise argparse.ArgumentError( + None, "--lblk-journal-only needs the journal disk named with " + "--blk-names or --blk-serials") if use_lblk and (args.pci_allowed or args.pci_blocked or getattr(args, 'device_model', '') or getattr(args, 'size_range', '') @@ -367,6 +383,7 @@ def main() -> None: "names_exclude": [x.strip() for x in args.blk_names_exclude.split(',') if x.strip()] or None, "serials": [x.strip() for x in args.blk_serials.split(',') if x.strip()] or None, "journal_device": bool(getattr(args, 'lblk_journal_device', False)), + "journal_only": bool(getattr(args, 'lblk_journal_only', False)), # Read before the discard below: the disks of an existing node # with a dedicated journal device are never partitioned. "keep_serials": sorted(utils.previous_lblk_dedicated_journal_serials()), diff --git a/tests/unit/test_lblk_journal_only.py b/tests/unit/test_lblk_journal_only.py new file mode 100644 index 0000000000..ccbf16991b --- /dev/null +++ b/tests/unit/test_lblk_journal_only.py @@ -0,0 +1,60 @@ +"""``sn configure --lblk-journal-only``: on a multi-attach cluster whose array +plugin creates each node's data LUN at add-node, the node's one local disk is +its journal and nothing else (never partitioned, no local data device).""" +import unittest + +from simplyblock_core import utils + + +def _disk(name="sdb", **kw): + return dict({"name": name, "serial": f"S-{name}", "size": 50 * 2**30, "type": "disk"}, **kw) + + +class TestJournalOnlyLayout(unittest.TestCase): + + def test_single_disk_becomes_flagged_journal(self): + out = utils.lblk_journal_only_layout({"sdb": _disk()}) + self.assertEqual(list(out), ["sdb"]) + self.assertTrue(out["sdb"]["journal"]) + self.assertEqual(out["sdb"]["serial"], "S-sdb") + + def test_input_is_not_modified(self): + entries = {"sdb": _disk()} + utils.lblk_journal_only_layout(entries) + self.assertNotIn("journal", entries["sdb"]) + + def test_two_disks_refused(self): + with self.assertRaisesRegex(ValueError, "exactly one disk"): + utils.lblk_journal_only_layout({"sdb": _disk(), "sdc": _disk("sdc")}, socket=1) + + def test_no_disk_refused(self): + with self.assertRaisesRegex(ValueError, "exactly one disk"): + utils.lblk_journal_only_layout({}) + + def test_partition_refused(self): + with self.assertRaisesRegex(ValueError, "whole disk"): + utils.lblk_journal_only_layout({"sdb1": _disk("sdb1", type="part")}) + + +class TestConfigureArgs(unittest.TestCase): + """The flag is in the CLI reference and the generated CLI.""" + + def test_flag_is_known_to_the_cli_reference(self): + import os + + import yaml + path = os.path.join(os.path.dirname(utils.__file__), "..", "..", "simplyblock_cli", "cli-reference.yaml") + with open(path, encoding="utf-8") as f: + text = f.read() + self.assertIn("--lblk-journal-only", text) + ref = yaml.safe_load(text) + self.assertTrue(ref) + + def test_generated_cli_has_the_flag(self): + from simplyblock_cli import cli + with open(cli.__file__, encoding="utf-8") as f: + self.assertIn("'--lblk-journal-only'", f.read()) + + +if __name__ == "__main__": + unittest.main() From 6b83da7da044150581866bf69163b1b2893dca20 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 16:33:23 +0100 Subject: [PATCH 13/50] fix(multi-attach): node agent accepts a journal-only lblk node config validate_node_config (run by the node agent when add-node reads the node's config) still required two lblk entries, so a node configured with --lblk-journal-only was refused at add-node ("Please run sbcli sn configure before adding the storage node"; agent log: "lblk mode requires at least 2 partitions or SSDs per node; node 0 carries 1"). Found on the multi-attach bed. A single entry is now valid when it is the flagged journal; add-node checks the minimum again once the array LUN is added. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/utils/__init__.py | 6 +++++- tests/unit/test_lblk_eligibility.py | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 894d0572c0..bf975fa98d 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -3676,7 +3676,11 @@ def validate_node_config(node): f"'size' in node: {node.get('socket')}") return False if lblk_devices: - if len(lblk_devices) < constants.LBLK_MIN_DEVICES_PER_NODE: + # sn configure --lblk-journal-only: the node's one entry is its flagged + # journal disk; its data device is the array LUN added at add-node, + # where the minimum is checked again with that LUN counted. + journal_only = len(lblk_devices) == 1 and bool(lblk_devices[0].get("journal")) + if len(lblk_devices) < constants.LBLK_MIN_DEVICES_PER_NODE and not journal_only: logger.error( f"lblk mode requires at least {constants.LBLK_MIN_DEVICES_PER_NODE} " f"partitions or SSDs per node; node {node.get('socket')} carries " diff --git a/tests/unit/test_lblk_eligibility.py b/tests/unit/test_lblk_eligibility.py index da3999fe0e..bbe25e35a3 100644 --- a/tests/unit/test_lblk_eligibility.py +++ b/tests/unit/test_lblk_eligibility.py @@ -356,6 +356,12 @@ def test_single_lblk_entry_rejected(self): node = self._node(lblk_devices=[{"name": "sdb", "serial": "S1", "size": 100}]) self.assertFalse(utils.validate_node_config(node)) + def test_single_journal_only_entry_valid(self): + # sn configure --lblk-journal-only: the data device is the array LUN + # added at add-node, which checks the minimum again with it counted. + node = self._node(lblk_devices=[{"name": "sdb", "serial": "S1", "size": 100, "journal": True}]) + self.assertTrue(utils.validate_node_config(node)) + def test_two_journal_flags_rejected(self): node = self._node(lblk_devices=[ {"name": "sdb1", "serial": "S1", "size": 100, "journal": True}, From 5e0197e67d595670aed8acf3b05b99006aee7707 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 16:47:17 +0100 Subject: [PATCH 14/50] fix(multi-attach): find the EC2 instance of a node by its private IP Found on the multi-attach bed: add-node failed with "aws-ebs: host ip-172-31-36-102 has no EC2 instance id". Two causes: - Docker deployments start the node agent with WITHOUT_CLOUD_INFO=True (startup time, f93a3c737), so the agent never reports the instance. - _amazon_info called ec2_metadata.EC2Metadata(session=...), which the installed ec2_metadata no longer accepts (TypeError), so it returned None on every AWS instance even where the probe runs. The aws-ebs plugin now resolves a host without an instance id by the node's private IP (exactly one pending/running instance in the volumes' zone); host_initiators carries the IP, from add-node's node address and from StorageNode.mgmt_ip. _amazon_info reads the identity document over IMDSv2 directly. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/array/aws_ebs.py | 23 ++++++++++- .../controllers/array_controller.py | 17 ++++---- simplyblock_core/storage_node_ops.py | 3 +- simplyblock_web/api/internal/_node_info.py | 9 ++-- tests/unit/test_array_lifecycle.py | 41 +++++++++++++++++-- 5 files changed, 78 insertions(+), 15 deletions(-) diff --git a/simplyblock_core/array/aws_ebs.py b/simplyblock_core/array/aws_ebs.py index 3e43c4a58b..e68888f145 100644 --- a/simplyblock_core/array/aws_ebs.py +++ b/simplyblock_core/array/aws_ebs.py @@ -230,7 +230,11 @@ def _free_device_name(self, instance_id) -> str: def ensure_host(self, name, initiators): instance_id = (initiators or {}).get("ec2_instance_id") or "" if not instance_id.startswith("i-"): - raise ArrayError(f"aws-ebs: host {name} has no EC2 instance id") + # The node agent reports the instance id only when its cloud + # probe runs (docker deployments start it with + # WITHOUT_CLOUD_INFO). The node's private IP identifies the + # instance as well, within the zone of the volumes. + instance_id = self._instance_by_private_ip(name, (initiators or {}).get("ip") or "") try: res = self._client().describe_instances(InstanceIds=[instance_id]) inst = res["Reservations"][0]["Instances"][0] @@ -245,6 +249,23 @@ def ensure_host(self, name, initiators): f"{self.options.get('az')}; Multi-Attach volumes cannot cross zones") return instance_id + def _instance_by_private_ip(self, name, ip): + if not ip: + raise ArrayError(f"aws-ebs: host {name} has no EC2 instance id and no IP") + try: + res = self._client().describe_instances(Filters=[ + {"Name": "private-ip-address", "Values": [ip]}, + {"Name": "availability-zone", "Values": [self.options.get("az") or ""]}, + {"Name": "instance-state-name", "Values": ["pending", "running"]}]) + except Exception as e: + raise ArrayError(f"aws-ebs: find the instance of host {name} ({ip}): {self._err(e)}") from e + ids = [i["InstanceId"] for r in res.get("Reservations") or [] for i in r.get("Instances") or []] + if len(ids) != 1: + raise ArrayError( + f"aws-ebs: host {name} has no EC2 instance id, and {len(ids)} running instances " + f"in {self.options.get('az')} have the private IP {ip}") + return ids[0] + def create_lun(self, name, size_bytes, tags): all_tags = self.cluster_tags(dict(self.options.get("tags") or {}, **(tags or {})), name) all_tags.setdefault("Name", name) diff --git a/simplyblock_core/controllers/array_controller.py b/simplyblock_core/controllers/array_controller.py index 7b85d00ebd..7619b88072 100644 --- a/simplyblock_core/controllers/array_controller.py +++ b/simplyblock_core/controllers/array_controller.py @@ -249,11 +249,13 @@ def mutate(fresh): # ------------------------------------------------------------------- hosts -def host_initiators(hostname="", cloud_instance=None, system_id="") -> dict: - """What identifies a host on an array: its hostname, system id and, on - AWS, the EC2 instance id (reported by the node agent from the instance - metadata service, IMDSv2).""" +def host_initiators(hostname="", cloud_instance=None, system_id="", ip="") -> dict: + """What identifies a host on an array: its hostname, system id, IP and, + on AWS, the EC2 instance id (reported by the node agent from the + instance metadata service when its cloud probe runs).""" out = {"hostname": hostname} + if ip: + out["ip"] = ip if system_id: out["system_id"] = system_id cloud = cloud_instance or {} @@ -264,7 +266,8 @@ def host_initiators(hostname="", cloud_instance=None, system_id="") -> dict: def _node_initiators(node) -> dict: cloud = {"cloud": getattr(node, "cloud_name", ""), "id": getattr(node, "cloud_instance_id", "")} - return host_initiators(multi_attach.array_host(node), cloud, getattr(node, "system_uuid", "")) + return host_initiators(multi_attach.array_host(node), cloud, getattr(node, "system_uuid", ""), + getattr(node, "mgmt_ip", "") or "") def ensure_node_host(plugin, node, db=None) -> str: @@ -303,7 +306,7 @@ def _find_in_inventory(inventory, wwn): def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configured, - timeout=LUN_APPEAR_TIMEOUT_SEC): + timeout=LUN_APPEAR_TIMEOUT_SEC, node_ip=""): """Create (or find) the LUN of the node being added, present it to the node's host and wait until the host sees it. @@ -321,7 +324,7 @@ def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configur try: plugin = array.plugin_for_cluster(cluster) host_id = plugin.ensure_host(hostname, host_initiators( - hostname, node_info.get("cloud_instance"), node_info.get("system_id", ""))) + hostname, node_info.get("cloud_instance"), node_info.get("system_id", ""), node_ip)) existing = next((e for e in lblk_configured or [] if e.get("array_lun_id")), None) if existing: lun = {"id": existing["array_lun_id"], "wwn": existing.get("lun_wwn", "")} diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 99654c59e3..9f26e8f3a7 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4444,7 +4444,8 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, return False try: lun_entry, array_host_id = array_controller.provision_node_lun( - cluster, snode_api, node_info, node_config, lblk_configured) + cluster, snode_api, node_info, node_config, lblk_configured, + node_ip=node_addr.split(":")[0]) except array_controller.ArrayLifecycleError as e: logger.error(f"Could not provision the node's array LUN: {e}") return False diff --git a/simplyblock_web/api/internal/_node_info.py b/simplyblock_web/api/internal/_node_info.py index ef57a7cf43..be3ccf06f0 100644 --- a/simplyblock_web/api/internal/_node_info.py +++ b/simplyblock_web/api/internal/_node_info.py @@ -158,10 +158,13 @@ def _google_info() -> dict | None: def _amazon_info() -> dict | None: + # IMDSv2 directly: ec2_metadata.EC2Metadata no longer takes a session + # (TypeError, so this returned None on every AWS instance). try: - import ec2_metadata - session = requests.session() - data = ec2_metadata.EC2Metadata(session=session).instance_identity_document # type: ignore[call-arg] + token = requests.put("http://169.254.169.254/latest/api/token", timeout=2, + headers={"X-aws-ec2-metadata-token-ttl-seconds": "60"}).text + data = requests.get("http://169.254.169.254/latest/dynamic/instance-identity/document", + timeout=2, headers={"X-aws-ec2-metadata-token": token}).json() return { "id": data["instanceId"], "type": data["instanceType"], diff --git a/tests/unit/test_array_lifecycle.py b/tests/unit/test_array_lifecycle.py index 2ffd6a4ea2..e1cb2f6f5e 100644 --- a/tests/unit/test_array_lifecycle.py +++ b/tests/unit/test_array_lifecycle.py @@ -861,6 +861,33 @@ def test_ensure_host_checks_zone(self): with self.assertRaisesRegex(array.ArrayError, "no EC2 instance id"): self.p.ensure_host("w3", {"hostname": "w3"}) + def test_ensure_host_by_private_ip(self): + # Docker deployments start the node agent with WITHOUT_CLOUD_INFO, so + # the instance is found by the node's private IP within the zone. + expected = {"Filters": [ + {"Name": "private-ip-address", "Values": ["172.31.36.102"]}, + {"Name": "availability-zone", "Values": ["us-east-1a"]}, + {"Name": "instance-state-name", "Values": ["pending", "running"]}]} + self.stub.add_response("describe_instances", {"Reservations": [{"Instances": [{ + "InstanceId": "i-7"}]}]}, expected) + self.stub.add_response("describe_instances", {"Reservations": [{"Instances": [{ + "InstanceId": "i-7", "Placement": {"AvailabilityZone": "us-east-1a"}}]}]}) + self.assertEqual(self.p.ensure_host("w1", {"hostname": "w1", "ip": "172.31.36.102"}), "i-7") + self.stub.assert_no_pending_responses() + + def test_ensure_host_by_ip_refuses_none_or_several(self): + self.stub.add_response("describe_instances", {"Reservations": []}) + with self.assertRaisesRegex(array.ArrayError, "0 running instances"): + self.p.ensure_host("w1", {"ip": "10.0.0.9"}) + self.stub.add_response("describe_instances", {"Reservations": [ + {"Instances": [{"InstanceId": "i-1"}, {"InstanceId": "i-2"}]}]}) + with self.assertRaisesRegex(array.ArrayError, "2 running instances"): + self.p.ensure_host("w1", {"ip": "10.0.0.9"}) + + def test_host_initiators_carry_the_ip(self): + out = array_controller.host_initiators("w1", {}, "", "172.31.36.102") + self.assertEqual(out, {"hostname": "w1", "ip": "172.31.36.102"}) + def test_connect(self): p = aws_ebs.AwsEbsPlugin(cluster_id=CID) with self.assertRaisesRegex(array.ArrayError, "'az'"): @@ -970,12 +997,20 @@ def test_amazon_info_reports_the_zone(self): from simplyblock_web.api.internal import _node_info doc = {"instanceId": "i-1", "instanceType": "m5", "privateIp": "10.0.0.1", "availabilityZone": "us-east-1a", "region": "us-east-1"} - fake = MagicMock() - fake.EC2Metadata.return_value.instance_identity_document = doc - with patch.dict("sys.modules", {"ec2_metadata": fake}): + token = MagicMock(text="tok") + ident = MagicMock() + ident.json.return_value = doc + with patch.object(_node_info.requests, "put", return_value=token) as put, patch.object(_node_info.requests, "get", return_value=ident) as get: info = _node_info._amazon_info() self.assertEqual((info["id"], info["availability_zone"], info["region"]), ("i-1", "us-east-1a", "us-east-1")) + self.assertIn("/latest/api/token", put.call_args.args[0]) + self.assertEqual(get.call_args.kwargs["headers"], {"X-aws-ec2-metadata-token": "tok"}) + + def test_amazon_info_none_off_aws(self): + from simplyblock_web.api.internal import _node_info + with patch.object(_node_info.requests, "put", side_effect=OSError("no route")): + self.assertIsNone(_node_info._amazon_info()) def test_rescan_dm_multipath_map(self): from simplyblock_web import node_utils From 8cbc2743c09f0124f4e5fbbae1be1fc2387c3383 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 17:08:24 +0100 Subject: [PATCH 15/50] fix(multi-attach): export LUN namespaces with an explicit NGUID Found on the multi-attach bed: activation failed because the owner's loopback controller to its own LUN could not register its namespace bdev ("Unable to add uuid:e6d24bda-... alias for bdev remote_alceml_n1"). The namespace was added with the derived UUID, but SPDK filled the NGUID from the exported PT bdev's UUID, and the host side registers the namespace bdev under that NGUID. The ptnonexcl bdev derives its UUID from its name (bdev_ptnonexcl.c, uuid5 of "_PT"), which is the same on every node, so on any node that both holds the PT bdev and consumes the LUN (the owner's loopback, a takeover node) the two UUIDs collide. Multi-attach namespaces are now added with nguid = the derived namespace UUID's hex digits, at node add, device restart and takeover. Regular devices call the RPC exactly as before. Co-Authored-By: Claude Opus 5.5 --- .../controllers/device_controller.py | 3 +- simplyblock_core/controllers/lun_takeover.py | 3 +- simplyblock_core/multi_attach.py | 18 ++++++++++++ simplyblock_core/storage_node_ops.py | 3 +- tests/unit/test_lun_takeover.py | 4 +-- tests/unit/test_multi_attach.py | 29 +++++++++++++++++-- 6 files changed, 53 insertions(+), 7 deletions(-) diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index ef33181175..b3bec39bdc 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -847,7 +847,8 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False): # Every node that exports the LUN must use the same namespace # UUID: SPDK multipath joins paths by namespace identity. ret = rpc_client.nvmf_subsystem_add_ns( - subsystem_nqn, pt_name, multi_attach.namespace_uuid(device_obj)) + subsystem_nqn, pt_name, multi_attach.namespace_uuid(device_obj), + nguid=multi_attach.namespace_nguid(device_obj)) else: ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name) else: diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index 3f9e9976a6..0f73035e73 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -441,7 +441,8 @@ def _export_on_standby(db, cluster, standby, dev, path, deadline) -> dict: if not ips: raise TakeoverError(f"node {sid} has no data IP to export the LUN on") if not any(ns.get("name") == pt for ns in (subsys.get("namespaces") or [])): - if not rpc.nvmf_subsystem_add_ns(nqn, pt, multi_attach.namespace_uuid(dev)): + if not rpc.nvmf_subsystem_add_ns(nqn, pt, multi_attach.namespace_uuid(dev), + nguid=multi_attach.namespace_nguid(dev)): raise TakeoverError(f"namespace {pt} not added to {nqn} on {sid}") except Exception: _teardown(rpc, sid, created) diff --git a/simplyblock_core/multi_attach.py b/simplyblock_core/multi_attach.py index d7d3f07a27..c0ee063e21 100644 --- a/simplyblock_core/multi_attach.py +++ b/simplyblock_core/multi_attach.py @@ -71,6 +71,24 @@ def namespace_uuid(device) -> str: return str(uuid_mod.uuid5(_LUN_NS_UUID_NAMESPACE, device.get_id())) +def namespace_nguid(device): + """The NGUID a multi-attach device's namespace is exported with: the + namespace UUID's 32 hex digits. None for a regular device (SPDK then + derives it from the bdev, as before). + + Without it SPDK fills the NGUID from the exported PT bdev's UUID, and the + host side registers its namespace bdev under that NGUID. The ptnonexcl PT + bdev's UUID is derived from its name (bdev_ptnonexcl.c), the same name on + every node, so on any node that both holds the PT bdev and consumes the + LUN (the owner's loopback, a takeover node) the consumer bdev's UUID + collides with the PT bdev's and SPDK refuses to register it (found on the + multi-attach bed: "Unable to add uuid: alias for bdev + remote_alceml_n1").""" + if not is_multi_attach(device): + return None + return uuid_mod.UUID(namespace_uuid(device)).hex.upper() + + def cntlid_range(block: int) -> tuple[int, int]: """``(min_cntlid, max_cntlid)`` of controller-ID block ``block``. diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 9f26e8f3a7..ce01f3d2d3 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2095,7 +2095,8 @@ def _create_storage_device_stack(rpc_client, nvme, snode: StorageNode, after_res ip_list.append(iface.ip4_address) logger.info(f"add {pt_name} to subsystem") - ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name, multi_attach.namespace_uuid(nvme)) + ns_extra = {"nguid": multi_attach.namespace_nguid(nvme)} if multi_attach.is_multi_attach(nvme) else {} + ret = rpc_client.nvmf_subsystem_add_ns(subsystem_nqn, pt_name, multi_attach.namespace_uuid(nvme), **ns_extra) if not ret: logger.error(f"Failed to add: {pt_name} to the subsystem: {subsystem_nqn}") return None diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index 1ae3b837a0..70098f3396 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -84,10 +84,10 @@ def listeners_create(self, nqn, trtype, traddr, port): self.subsystems[nqn]["listen_addresses"].append({"traddr": traddr}) return True - def nvmf_subsystem_add_ns(self, nqn, pt, uuid=None): + def nvmf_subsystem_add_ns(self, nqn, pt, uuid=None, nguid=None): if not self._rec("nvmf_subsystem_add_ns", nqn, pt, uuid): return None - self.subsystems[nqn]["namespaces"].append({"name": pt, "uuid": uuid}) + self.subsystems[nqn]["namespaces"].append({"name": pt, "uuid": uuid, "nguid": nguid}) return 1 def alceml_set_qos_weights(self, weights): diff --git a/tests/unit/test_multi_attach.py b/tests/unit/test_multi_attach.py index 697455463a..e62adc62db 100644 --- a/tests/unit/test_multi_attach.py +++ b/tests/unit/test_multi_attach.py @@ -446,7 +446,8 @@ def test_multi_attach_device(self): nqn = f"{constants.CLUSTER_NQN}:lun:{DEV_ID}" rpc.subsystem_create.assert_called_once_with(nqn, "sbcli-cn", DEV_ID, min_cntlid=129, max_cntlid=192) rpc.nvmf_subsystem_add_ns.assert_called_once_with(nqn, f"alceml_{DEV_ID}_PT", - multi_attach.namespace_uuid(dev)) + multi_attach.namespace_uuid(dev), + nguid=multi_attach.namespace_nguid(dev)) self.assertEqual(snode.create_alceml.call_args.kwargs["stuck_io_timeout_ms"], 60000) self.assertEqual(dev.nvmf_nqn, nqn) @@ -481,7 +482,8 @@ def test_multi_attach_device(self): nqn = f"{constants.CLUSTER_NQN}:lun:{DEV_ID}" rpc.subsystem_create.assert_called_once_with(nqn, "sbcli-cn", DEV_ID, min_cntlid=129, max_cntlid=192) rpc.nvmf_subsystem_add_ns.assert_called_once_with(nqn, f"alceml_{DEV_ID}_PT", - multi_attach.namespace_uuid(dev)) + multi_attach.namespace_uuid(dev), + nguid=multi_attach.namespace_nguid(dev)) self.assertEqual(snode.create_alceml.call_args.kwargs["stuck_io_timeout_ms"], 60000) self.assertEqual(dev.nvmf_nqn, nqn) @@ -619,3 +621,26 @@ def test_connect_skips_own_regular_devices(self): if __name__ == "__main__": unittest.main() + + +class TestNamespaceNguid(unittest.TestCase): + """The NGUID of a multi-attach namespace is set explicitly: SPDK would + otherwise fill it from the PT bdev's name-derived UUID, which collides on + any node that also consumes the LUN.""" + + def test_regular_device_has_none(self): + self.assertIsNone(multi_attach.namespace_nguid(_device())) + + def test_multi_attach_nguid_is_the_namespace_uuid(self): + import uuid as uuid_mod + dev = _device(multi=True) + nguid = multi_attach.namespace_nguid(dev) + self.assertEqual(nguid, uuid_mod.UUID(multi_attach.namespace_uuid(dev)).hex.upper()) + self.assertEqual(len(nguid), 32) + + def test_differs_from_the_pt_bdev_uuid(self): + # bdev_ptnonexcl.c: uuid5(7e25812e-...-000000000001, "_PT"). + import uuid as uuid_mod + dev = _device(multi=True) + pt = uuid_mod.uuid5(uuid_mod.UUID("7e25812e-c8c0-4d3f-8599-000000000001"), f"alceml_{DEV_ID}_PT") + self.assertNotEqual(multi_attach.namespace_nguid(dev), pt.hex.upper()) From ad4b0c66c7cfda31734ca978e28b7a5d779cbf89 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 17:42:10 +0100 Subject: [PATCH 16/50] fix(multi-attach): a lost node degrades a multi-attach cluster, not suspends Found on the multi-attach bed: the first storage node loss suspended the cluster at the moment the node went offline. The suspend rule counts lost nodes against the erasure-coding parity, k = npcs, which is 0 on a 1+0 multi-attach cluster. The suspend then shut the healthy nodes down, the LUN takeover's standby among them, so the takeover (fenced in 3 s, standby found) failed at the export with "Could not reach remote". On a multi-attach cluster a lost node's LUN is taken over by a standby, so the node-loss tolerance is the cluster's FTT (max_fault_tolerance): with FTT1 one lost node is DEGRADED, two are SUSPENDED (a LUN is attached to two nodes). Regular clusters, 1+0 included, keep k = npcs. Co-Authored-By: Claude Opus 5.5 --- .../services/storage_node_monitor.py | 7 +++ .../unit/test_multi_attach_cluster_status.py | 52 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 tests/unit/test_multi_attach_cluster_status.py diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index b4adfedaf9..abe860e999 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -687,6 +687,13 @@ def _status_verdict_rules(cluster, snodes, affected_physical_nodes, online_nodes # npcs k = 1 n = cluster.distr_ndcs k = cluster.distr_npcs + if getattr(cluster, "multi_attach", False) is True: + # Multi-attach array LUNs run 1+0: no parity, but a lost node's LUN is + # taken over by a standby (controllers/lun_takeover), so the cluster + # survives as many lost nodes as its FTT. With k = npcs = 0 the first + # node loss suspended the cluster, which shut the other nodes down + # and with them the takeover's standby (found on the multi-attach bed). + k = max(k, int(getattr(cluster, "max_fault_tolerance", 0) or 1)) # Failure-domain-aware suspend criteria. When the cluster was created with # --enable-failure-domain, chunks are spread across distinct domains, so it diff --git a/tests/unit/test_multi_attach_cluster_status.py b/tests/unit/test_multi_attach_cluster_status.py new file mode 100644 index 0000000000..2da11818f9 --- /dev/null +++ b/tests/unit/test_multi_attach_cluster_status.py @@ -0,0 +1,52 @@ +"""Cluster status on a multi-attach (1+0 array LUN) cluster: a lost node is +covered by the LUN takeover, so the node-loss tolerance is the cluster's FTT, +not its (zero) parity. Found on the multi-attach bed: the first node loss +suspended the cluster, which shut the other nodes down, takeover standby +included.""" +import unittest +from unittest.mock import MagicMock + +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.services import storage_node_monitor as mon + + +def _cluster(multi_attach=True, ftt=1, ndcs=1, npcs=0): + return MagicMock(multi_attach=multi_attach, max_fault_tolerance=ftt, distr_ndcs=ndcs, + distr_npcs=npcs, enable_failure_domain=False, + strict_node_anti_affinity=False, migration_ft_impact=None) + + +def _verdict(cluster, affected, online_nodes=4, online_devices=4): + return mon._status_verdict_rules(cluster, [], set(range(affected)), online_nodes, + online_devices, []) + + +class TestMultiAttachNodeTolerance(unittest.TestCase): + + def test_healthy_is_active(self): + self.assertEqual(_verdict(_cluster(), 0), Cluster.STATUS_ACTIVE) + + def test_one_lost_node_with_ftt1_is_degraded(self): + self.assertEqual(_verdict(_cluster(), 1, online_nodes=3), Cluster.STATUS_DEGRADED) + + def test_two_lost_nodes_with_ftt1_suspend(self): + # each LUN is attached to two nodes only + self.assertEqual(_verdict(_cluster(), 2, online_nodes=2), Cluster.STATUS_SUSPENDED) + + def test_two_lost_nodes_with_ftt2_is_degraded(self): + self.assertEqual(_verdict(_cluster(ftt=2), 2, online_nodes=2), Cluster.STATUS_DEGRADED) + + def test_regular_no_parity_cluster_unchanged(self): + # 1+0 without multi-attach: no takeover, a lost node suspends as before + self.assertEqual(_verdict(_cluster(multi_attach=False), 1, online_nodes=3), + Cluster.STATUS_SUSPENDED) + + def test_regular_parity_cluster_unchanged(self): + self.assertEqual(_verdict(_cluster(multi_attach=False, ndcs=2, npcs=1), 1, online_nodes=3), + Cluster.STATUS_DEGRADED) + self.assertEqual(_verdict(_cluster(multi_attach=False, ndcs=2, npcs=1), 2, online_nodes=2), + Cluster.STATUS_SUSPENDED) + + +if __name__ == "__main__": + unittest.main() From 02caa0b25b14ffff4c52e65b36d3126e3dbe1031 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 13 Nov 2026 19:21:19 +0100 Subject: [PATCH 17/50] fix(multi-attach): take a LUN over once its node is unreachable, not offline Found on the multi-attach bed (F1, hard power-off of an owner under fio): the takeover started 6 minutes after the power-off, when the node was finally declared OFFLINE, long after the peers' 120 s hold, so the client got EIO. The OFFLINE verdict waits for every online peer to report the node's journal disconnected; the node was UNREACHABLE 20 s after the power-off. - The takeover of the LUNs a node serves starts once the node has been UNREACHABLE for MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC (15 s). The fence at the array comes first, so a node alive behind a management-network fault is cut off from the LUN before the standby opens it. The timer restarts with every new UNREACHABLE episode. - MULTI_ATTACH_HOLD_SEC 120 -> 150: it must cover detection (~20 s), the trigger delay and the takeover deadline (90 s). - A node that comes back ONLINE without a restart (a blip long enough to start a takeover) retires what it still holds of the LUNs taken over from it (lun_takeover.retire_stale_local_stack): its own loopback path in the LUN's controller, then the stale subsystem, PT and alceml. Nothing is formatted. A restart never builds these stacks. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/constants.py | 12 ++- simplyblock_core/controllers/lun_takeover.py | 55 ++++++++++++ .../services/storage_node_monitor.py | 51 +++++++++++ tests/unit/test_lun_takeover.py | 88 +++++++++++++++++++ tests/unit/test_multi_attach.py | 3 +- 5 files changed, 207 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 17d8887994..bfd5ed72d6 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -1192,7 +1192,10 @@ def get_config_var(name, default=None): # path is available (per-controller ctrlr_loss_timeout_sec), so a takeover # that adds the LUN's new export as a path within the window resumes I/O # instead of failing it. The cluster-wide bdev_nvme defaults (1 s) stay. -MULTI_ATTACH_HOLD_SEC = 120 +# It must cover: detecting the owner unreachable (~20 s), the +# MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC confirmation, and the takeover up to +# its MULTI_ATTACH_TAKEOVER_DEADLINE_SEC. +MULTI_ATTACH_HOLD_SEC = 150 MULTI_ATTACH_RECONNECT_DELAY_SEC = 2 MULTI_ATTACH_FAST_IO_FAIL_SEC = MULTI_ATTACH_HOLD_SEC # The alceml of a multi-attach device fails its pending I/O and unregisters @@ -1212,6 +1215,13 @@ def get_config_var(name, default=None): # peers' hold, or their I/O fails before the new path exists; it gives up at # this deadline (seconds from its start) and the device goes unavailable. MULTI_ATTACH_TAKEOVER_DEADLINE_SEC = 90 +# The takeover of a node's LUNs starts once the node has been UNREACHABLE +# this long, not when it is declared OFFLINE: the OFFLINE verdict waits for +# every online peer to see the node's journal disconnected, which took about +# six minutes on the multi-attach bed, far past the peers' hold. The fence +# comes first, so a takeover of a node that is alive behind a management +# network fault is safe; it costs a short I/O pause on that LUN. +MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC = 15 # Lease of a running takeover (one per LUN, in FDB): a holder older than this # is presumed dead. MULTI_ATTACH_TAKEOVER_LOCK_TTL_SEC = 2 * MULTI_ATTACH_TAKEOVER_DEADLINE_SEC diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index 0f73035e73..c0ab1e9038 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -566,6 +566,61 @@ def _switch_paths(db, cluster_id, dev, standby, failed, failed_paths) -> list: return added +def retire_stale_local_stack(node) -> list: + """Retire, on a node that is back ONLINE, what it still holds of the LUNs + another node took over while it was unreachable: its own path in each + ``remote_alceml_`` controller (the loopback to its now-stale export), + then that export's subsystem, PT bdev and alceml. Nothing is formatted; the + base bdev over the LUN is left alone (the node is fenced from the LUN). + + Needed when the node returns without a restart (a management-plane blip + long enough to start a takeover): a restart never builds these stacks + (``multi_attach.served_elsewhere``), but a node that never restarted still + has them, and its own distribs would keep writing through its loopback + into an alceml whose LUN the array has fenced. Returns the ids of the + devices retired; best-effort and idempotent.""" + from simplyblock_core.db_controller import DBController + db = DBController() + node_id = node.get_id() + try: + rpc = node.rpc_client(timeout=10, retry=1) + except Exception as e: # noqa: BLE001 + logger.warning("LUN retire: no RPC client for %s: %s", node_id, e) + return [] + retired = [] + for owner in db.get_storage_nodes_by_cluster_id(node.cluster_id): + for dev in owner.nvme_devices or []: + if not multi_attach.is_multi_attach(dev) or not multi_attach.served_elsewhere(dev, node_id): + continue + # Only a node that once exported the LUN can hold a stale stack: + # its owner, or a node it was taken over by before. + if node_id != dev.node_id and node_id not in (dev.node_paths or {}): + continue + pt = f"{dev.alceml_bdev}_PT" + try: + has_stack = bool(rpc.bdev_get(dev.alceml_bdev)) or bool(rpc.bdev_get(pt)) + except Exception: # noqa: BLE001 + has_stack = False + if not has_stack: + continue + name = f"remote_{dev.alceml_bdev}" + own_ips = [iface.ip4_address for iface in node.data_nics or [] if iface.ip4_address] + for ip in own_ips: + try: + rpc.bdev_nvme_detach_controller(name, traddr=ip, trsvcid=node.nvmf_port, + trtype=_transport(node) or "TCP") + except Exception as e: # noqa: BLE001 + logger.info("LUN retire: own path %s of %s on %s not detached: %s", + ip, name, node_id, e) + nqn = multi_attach.device_subsystem_nqn(node, dev, db.get_cluster_by_id(node.cluster_id)) + _teardown(rpc, node_id, [("alceml", dev.alceml_bdev), ("pt", pt), ("subsystem", nqn)]) + logger.warning("LUN retire: %s on %s is served by %s since a takeover; its stale " + "local stack was removed (nothing formatted)", + dev.get_id(), node_id, dev.serving_node()) + retired.append(dev.get_id()) + return retired + + # ------------------------------------------------------------- operator def unfence_node(device_id, node_id) -> bool: diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index abe860e999..a14c0b9841 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -1978,6 +1978,34 @@ def set_node_offline(node): logger.error("add_node_to_auto_restart for %s failed: %s", node.get_id(), ar_e) +# node id -> monotonic time it was first seen UNREACHABLE in the current +# episode (multi-attach LUN takeover trigger; see +# constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC). Cleared on any other status. +_unreachable_since: dict = {} + + +def _take_over_luns_of_unreachable(node): + """Multi-attach clusters (only): start the takeover of the LUNs a node + serves once it has been UNREACHABLE for MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC. + The takeover fences the node at the array first; each LUN gets one + thread (_start_lun_takeover is idempotent across monitor cycles).""" + node_id = node.get_id() + since = _unreachable_since.setdefault(node_id, time.monotonic()) + if time.monotonic() - since < constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC: + return + try: + cluster = db.get_cluster_by_id(node.cluster_id) + except Exception as e: # noqa: BLE001 + logger.warning("Cluster of %s not read (%s); no LUN takeover", node_id, e) + return + if not multi_attach.is_multi_attach(cluster): + return + for _owner, dev in multi_attach.devices_served_by( + db.get_storage_nodes_by_cluster_id(node.cluster_id), node_id): + if dev.status in _DEV_SERVING_STATES: + _start_lun_takeover(dev.get_id(), node_id, "node unreachable") + + def set_node_unreachable(node): # Same rationale as set_node_offline: when the runner owns the node's # status transitions (IN_SHUTDOWN → OFFLINE → IN_RESTART → ONLINE), the @@ -1992,6 +2020,11 @@ def set_node_unreachable(node): logger.debug("Setting node to UNREACHABLE state failed") logger.error(e) + try: + _take_over_luns_of_unreachable(node) + except Exception as e: # noqa: BLE001 + logger.error("LUN takeover trigger for unreachable node %s failed: %s", node.get_id(), e) + # Check data-plane health from surviving nodes. If all online peers # report the unreachable node's remote JM as disconnected, the data # plane is truly down and we can escalate to offline. @@ -2816,6 +2849,10 @@ def _spdk_is_dead(snode): def check_node(snode): snode = db.get_storage_node_by_id(snode.get_id()) + if snode.status != StorageNode.STATUS_UNREACHABLE: + # Ends an UNREACHABLE episode: a later one starts the multi-attach + # LUN takeover timer afresh (_take_over_luns_of_unreachable). + _unreachable_since.pop(snode.get_id(), None) if snode.status == StorageNode.STATUS_ONLINE: storage_network.publish_ports(snode, at_most_every=PORT_PUBLISH_INTERVAL_SEC) @@ -3094,6 +3131,20 @@ def check_node(snode): ) storage_node_ops.set_node_status(snode.get_id(), StorageNode.STATUS_ONLINE) readmit_devices_after_node_online(snode.get_id()) + _retire_taken_over_luns(snode) + + +def _retire_taken_over_luns(snode): + """Multi-attach clusters (only): a node back ONLINE without a restart + drops what it still holds of LUNs taken over while it was unreachable + (lun_takeover.retire_stale_local_stack).""" + try: + if not multi_attach.is_multi_attach(db.get_cluster_by_id(snode.cluster_id)): + return + from simplyblock_core.controllers import lun_takeover + lun_takeover.retire_stale_local_stack(snode) + except Exception as e: # noqa: BLE001 + logger.error("Retiring taken-over LUN stacks on %s failed: %s", snode.get_id(), e) def readmit_devices_after_node_online(node_id): diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index 70098f3396..b48fd2dfb1 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -702,3 +702,91 @@ def test_takeover_cause_needs_an_online_serving_node(self): if __name__ == "__main__": unittest.main() + + +# --------------------------------------------- multi-attach bed findings, F1 + +class TestUnreachableTrigger(unittest.TestCase): + """The takeover starts once a node has been UNREACHABLE for + MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC, not when it is declared OFFLINE: + on the bed the OFFLINE verdict took ~6 min, far past the peers' hold.""" + + def setUp(self): + from simplyblock_core.services import storage_node_monitor as mon + self.mon = mon + mon._unreachable_since.clear() + self.addCleanup(mon._unreachable_since.clear) + self.db, self.dev = _bed() + self.db.get_storage_node_by_id("node-a").status = StorageNode.STATUS_UNREACHABLE + for target, attr, kw in ((mon, "db", {"new": self.db}), + (mon, "_start_lun_takeover", {})): + p = patch.object(target, attr, **kw) + started = p.start() + if attr == "_start_lun_takeover": + self.start = started + self.addCleanup(p.stop) + + def _trigger(self, after): + node = self.db.get_storage_node_by_id("node-a") + with patch.object(self.mon.time, "monotonic", return_value=1000.0): + self.mon._take_over_luns_of_unreachable(node) + with patch.object(self.mon.time, "monotonic", return_value=1000.0 + after): + self.mon._take_over_luns_of_unreachable(node) + + def test_not_before_the_confirmation_time(self): + self._trigger(constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC - 1) + self.start.assert_not_called() + + def test_after_the_confirmation_time(self): + self._trigger(constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC) + self.start.assert_called_once_with(DEV_ID, "node-a", "node unreachable") + + def test_regular_cluster_never(self): + self.db.cluster.multi_attach = False + self._trigger(10 * constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC) + self.start.assert_not_called() + + def test_hold_covers_detection_trigger_and_takeover(self): + # ~20 s to see the node unreachable, the trigger delay, the takeover deadline + self.assertGreater(constants.MULTI_ATTACH_HOLD_SEC, + 20 + constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC + + constants.MULTI_ATTACH_TAKEOVER_DEADLINE_SEC) + + +class TestRetireStaleLocalStack(unittest.TestCase): + """A node back ONLINE without a restart drops its stale stack of a LUN + another node took over: own loopback path first, then subsystem, PT, + alceml. Nothing is formatted.""" + + def _setup(self, served_by="node-b", with_stack=True): + db, dev = _bed() + a = db.get_storage_node_by_id("node-a") + a.status = StorageNode.STATUS_ONLINE + dev.serving_node_id = served_by + if with_stack: + a.spdk.bdevs[dev.alceml_bdev] = {"name": dev.alceml_bdev} + a.spdk.bdevs[f"{dev.alceml_bdev}_PT"] = {"name": f"{dev.alceml_bdev}_PT"} + with patch("simplyblock_core.db_controller.DBController", return_value=db): + retired = lun_takeover.retire_stale_local_stack(a) + return a, dev, retired + + def test_retires_in_order(self): + a, dev, retired = self._setup() + self.assertEqual(retired, [DEV_ID]) + detach = a.spdk.ops("detach") + self.assertEqual(detach[0][1][0], f"remote_{dev.alceml_bdev}") + self.assertEqual(detach[0][2]["traddr"], "10.0.0.1") + order = [c[0] for c in a.spdk.calls if c[0] in ("subsystem_delete", "bdev_PT_NoExcl_delete", + "bdev_alceml_delete")] + self.assertEqual(order, ["subsystem_delete", "bdev_PT_NoExcl_delete", "bdev_alceml_delete"]) + self.assertFalse(a.spdk.ops("bdev_alceml_create")) + + def test_nothing_when_still_served_here(self): + a, _dev, retired = self._setup(served_by="") + self.assertEqual(retired, []) + self.assertFalse(a.spdk.ops("subsystem_delete")) + + def test_nothing_without_a_stack(self): + a, _dev, retired = self._setup(with_stack=False) + self.assertEqual(retired, []) + self.assertFalse(a.spdk.ops("detach")) diff --git a/tests/unit/test_multi_attach.py b/tests/unit/test_multi_attach.py index e62adc62db..b24e4a526c 100644 --- a/tests/unit/test_multi_attach.py +++ b/tests/unit/test_multi_attach.py @@ -336,7 +336,8 @@ def test_kwargs_are_empty_for_regular_devices(self): def test_hold_and_stuck_io_constants(self): self.assertLess(constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS, constants.MULTI_ATTACH_HOLD_SEC * 1000) self.assertEqual(multi_attach.attach_controller_kwargs(_device(multi=True)), { - "ctrlr_loss_timeout_sec": 120, "reconnect_delay_sec": 2, "fast_io_fail_timeout_sec": 120}) + "ctrlr_loss_timeout_sec": constants.MULTI_ATTACH_HOLD_SEC, "reconnect_delay_sec": 2, + "fast_io_fail_timeout_sec": constants.MULTI_ATTACH_HOLD_SEC}) self.assertEqual(multi_attach.alceml_create_kwargs(_device(multi=True)), {"stuck_io_timeout_ms": 60000}) From c91eefc1c56cbd041f3ca774d86aa739f95cdef7 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 19 Nov 2026 12:13:31 +0100 Subject: [PATCH 18/50] fix(multi-attach): a fenced node that is back online becomes a standby again After a takeover, the failed node stayed fenced from its LUN until an operator called unfence_node, so a node that returned after an outage was never a standby again and a second failure of the serving node found no standby. The monitor now checks an ONLINE node that a takeover fenced (at most every 30 s) and unfences it at the array when nothing it held can still reach the LUN: the LUN is served by another ONLINE node, the node holds no alceml or PT bdev of it (a stale stack is retired first), and its host shows no disk of the LUN. A host that still shows the LUN (e.g. a multipath map still queueing) stays fenced for the operator. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/controllers/lun_takeover.py | 77 ++++++++++++++++++- simplyblock_core/multi_attach.py | 4 +- .../services/storage_node_monitor.py | 25 ++++++ tests/unit/test_lun_takeover.py | 54 +++++++++++++ 4 files changed, 155 insertions(+), 5 deletions(-) diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index c0ab1e9038..73dfe8ef0f 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -621,13 +621,84 @@ def retire_stale_local_stack(node) -> list: return retired +def reintegrate_fenced_node(node) -> list: + """Make a node that is back ONLINE a standby again for every LUN a + takeover fenced it from: unfence it at the array (re-present the LUN). + + Only when nothing it held can still reach the LUN late: + + * the LUN is served by another ONLINE node (the takeover completed; a + fenced node of a takeover still holding is never unfenced); + * the node holds no alceml or PT bdev of the device (its stale stack was + retired, or a restart never built it); + * the node's host shows no disk carrying the LUN (``lun_info`` empty): the + fence removed every path, so no host-side queue (multipath + ``queue_if_no_path`` included) holds writes for it. A host that still + shows the LUN keeps the fence until an operator calls + :func:`unfence_node`. + + Returns the ids of the devices unfenced; best-effort, idempotent.""" + db = DBController() + node_id = node.get_id() + if node.status != StorageNode.STATUS_ONLINE: + return [] + nodes = db.get_storage_nodes_by_cluster_id(node.cluster_id) + by_id = {n.get_id(): n for n in nodes} + todo = [dev for owner in nodes for dev in owner.nvme_devices or [] + if multi_attach.fenced_from(dev, node_id)] + if not todo: + return [] + try: + rpc = node.rpc_client(timeout=10, retry=1) + except Exception as e: # noqa: BLE001 + logger.info("LUN reintegrate: no RPC client for %s: %s", node_id, e) + return [] + done = [] + for dev in todo: + serving = by_id.get(dev.serving_node()) + if serving is None or serving.get_id() == node_id or serving.status != StorageNode.STATUS_ONLINE: + continue + try: + stale = bool(rpc.bdev_get(dev.alceml_bdev)) or bool(rpc.bdev_get(f"{dev.alceml_bdev}_PT")) + except Exception: # noqa: BLE001 + stale = False + if stale: + retire_stale_local_stack(node) + try: + stale = bool(rpc.bdev_get(dev.alceml_bdev)) or bool(rpc.bdev_get(f"{dev.alceml_bdev}_PT")) + except Exception: # noqa: BLE001 + stale = False + if stale: + logger.info("LUN reintegrate: %s still holds a stack of %s; stays fenced", + node_id, dev.get_id()) + continue + try: + result, err = node.client(timeout=15).lun_info(dev.lun_wwn) + except Exception as e: # noqa: BLE001 + logger.info("LUN reintegrate: agent of %s not reached: %s", node_id, e) + continue + if err or not isinstance(result, list): + logger.info("LUN reintegrate: LUN inventory of %s unavailable (%s)", node_id, err) + continue + if result: + logger.warning("LUN reintegrate: host of %s still shows LUN %s (%s); it stays fenced " + "until an operator unfences it", node_id, dev.lun_wwn, + [e.get("device_path") for e in result if isinstance(e, dict)]) + continue + if unfence_node(dev.get_id(), node_id): + logger.info("LUN reintegrate: %s is a standby of LUN %s again", node_id, dev.lun_wwn) + done.append(dev.get_id()) + return done + + # ------------------------------------------------------------- operator def unfence_node(device_id, node_id) -> bool: """Give ``node_id`` access to the LUN of ``device_id`` again and drop it - from ``fenced_node_ids``. Never called automatically: only once the - node's host is known to hold no queued writes for the LUN (its multipath - queues drained or the host rebooted). Returns True on success.""" + from ``fenced_node_ids``. Called by :func:`reintegrate_fenced_node` once + the node's host shows no disk of the LUN, else by an operator once the + host is known to hold no queued writes for it (its multipath queues + drained or the host rebooted). Returns True on success.""" db = DBController() try: dev = db.get_storage_device_by_id(device_id) diff --git a/simplyblock_core/multi_attach.py b/simplyblock_core/multi_attach.py index c0ee063e21..b10b43f3ed 100644 --- a/simplyblock_core/multi_attach.py +++ b/simplyblock_core/multi_attach.py @@ -212,8 +212,8 @@ def fenced_from(device, node_id) -> bool: def not_servable_here(device, node_id) -> bool: """``node_id`` must not build a stack for ``device``: another node serves its LUN, or ``node_id`` is fenced from it (the takeover after its failure - did not complete; an operator unfences it -- lun_takeover.unfence_node -- - once its host is known to hold no queued writes).""" + did not complete, or it has not been reintegrated yet -- + lun_takeover.reintegrate_fenced_node / unfence_node).""" return served_elsewhere(device, node_id) or fenced_from(device, node_id) diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index fb937f2e0b..ce6f1f60f6 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -2914,6 +2914,7 @@ def check_node(snode): _unreachable_since.pop(snode.get_id(), None) if snode.status == StorageNode.STATUS_ONLINE: storage_network.publish_ports(snode, at_most_every=PORT_PUBLISH_INTERVAL_SEC) + _reintegrate_fenced_luns(snode) # Self-heal a node stranded in IN_SHUTDOWN. A completed shutdown flips the # node to OFFLINE; if it is still IN_SHUTDOWN well past the grace window the @@ -3219,6 +3220,30 @@ def _retire_taken_over_luns(snode): logger.error("Retiring taken-over LUN stacks on %s failed: %s", snode.get_id(), e) +_reintegrate_last: dict = {} +REINTEGRATE_INTERVAL_SEC = 30 + + +def _reintegrate_fenced_luns(snode): + """Multi-attach clusters (only): an ONLINE node a takeover fenced from a + LUN becomes a standby of it again (lun_takeover.reintegrate_fenced_node), + checked at most every REINTEGRATE_INTERVAL_SEC per node.""" + node_id = snode.get_id() + if not any(multi_attach.fenced_from(d, node_id) + for n in db.get_storage_nodes_by_cluster_id(snode.cluster_id) + for d in n.nvme_devices or []): + return + now = time.monotonic() + if now - _reintegrate_last.get(node_id, 0) < REINTEGRATE_INTERVAL_SEC: + return + _reintegrate_last[node_id] = now + try: + from simplyblock_core.controllers import lun_takeover + lun_takeover.reintegrate_fenced_node(snode) + except Exception as e: # noqa: BLE001 + logger.error("Reintegrating fenced LUNs of %s failed: %s", node_id, e) + + def readmit_devices_after_node_online(node_id): """Re-admit a recovered node's devices right after it was set ONLINE. diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index b48fd2dfb1..9996827246 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -790,3 +790,57 @@ def test_nothing_without_a_stack(self): a, _dev, retired = self._setup(with_stack=False) self.assertEqual(retired, []) self.assertFalse(a.spdk.ops("detach")) + + +class TestReintegrateFencedNode(_TakeoverCase): + """A node a takeover fenced becomes a standby again (unfenced at the array) + once it is ONLINE, holds no stack of the LUN, and its host shows no disk of + the LUN; otherwise it stays fenced.""" + + def _taken_over(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + a = db.get_storage_node_by_id("node-a") + a.status = StorageNode.STATUS_ONLINE + a.agent.lun_info.return_value = ([], None) + return db, a + + def test_unfenced_when_host_shows_no_lun(self): + db, a = self._taken_over() + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), [DEV_ID]) + self.assertEqual(FakeArrayPlugin.calls[-1], ("unfence", WWN, "host-node-a")) + self.assertEqual(self.dev(db).fenced_node_ids, []) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + def test_stays_fenced_while_host_shows_lun(self): + db, a = self._taken_over() + a.agent.lun_info.return_value = ([_lun_entry()], None) + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + self.assertEqual(self.dev(db).fenced_node_ids, ["node-a"]) + + def test_stays_fenced_when_inventory_fails(self): + db, a = self._taken_over() + a.agent.lun_info.return_value = (None, "boom") + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + self.assertEqual(self.dev(db).fenced_node_ids, ["node-a"]) + + def test_stays_fenced_while_serving_node_is_down(self): + db, a = self._taken_over() + db.get_storage_node_by_id("node-b").status = StorageNode.STATUS_UNREACHABLE + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + self.assertEqual(self.dev(db).fenced_node_ids, ["node-a"]) + + def test_not_while_node_is_not_online(self): + db, a = self._taken_over() + a.status = StorageNode.STATUS_UNREACHABLE + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + + def test_stale_stack_is_retired_first(self): + db, a = self._taken_over() + dev = self.dev(db) + a.spdk.bdevs[dev.alceml_bdev] = {"name": dev.alceml_bdev} + a.spdk.bdevs[f"{dev.alceml_bdev}_PT"] = {"name": f"{dev.alceml_bdev}_PT"} + with patch("simplyblock_core.db_controller.DBController", return_value=db): + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), [DEV_ID]) + self.assertTrue(a.spdk.ops("bdev_alceml_delete")) + self.assertEqual(self.dev(db).fenced_node_ids, []) From fa04876abe84c874680808af4794f09fc0561b69 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 19 Nov 2026 13:37:21 +0100 Subject: [PATCH 19/50] fix(multi-attach): the owner never sweeps or repairs a LUN it does not serve 2026-11-19 FTT1 bed, SPDK container kill of a node serving two LUNs: the takeover fenced it (EBS detach), it restarted within 22 s, and the device monitor's lblk hot-removal sweep then found the fenced owner's LUN missing from its host and called device_remove. On a 1+0 cluster a removed device is lost stripes: read errors on every lvstore, nodes down, cluster suspended. - device_monitor: the presence sweep and the hung-IO check skip a multi-attach device that is not servable on the node (served elsewhere, or the node is fenced from it). - device_repair_due / device_repair: never rebuild such a device's stack on its owner. - unfence_node clears the fence record before the array call. The attachment reconcile detaches every host the record lists as fenced, so attaching first raced with it (the bed: attach at :09, reconcile detach at :10). With the record cleared the reconcile presents the LUN itself. Co-Authored-By: Claude Opus 5.5 --- .../controllers/device_controller.py | 10 ++++ simplyblock_core/controllers/lun_takeover.py | 22 ++++--- simplyblock_core/services/device_monitor.py | 18 +++++- tests/unit/test_lblk_watchdog.py | 58 +++++++++++++++++++ tests/unit/test_lun_takeover.py | 25 ++++++++ 5 files changed, 122 insertions(+), 11 deletions(-) diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index b3bec39bdc..76714cbb50 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -567,6 +567,11 @@ def device_repair(device_id, force=False): "Device %s was removed by operator request; not repairing", device_id) return False + if multi_attach.not_servable_here(device, device.node_id): + logger.info("Device %s: its LUN is served by node %s (owner fenced: %s); " + "the owner does not repair it", device_id, device.serving_node(), + multi_attach.fenced_from(device, device.node_id)) + return False if device.status != NVMeDevice.STATUS_ONLINE and device.retries_exhausted and not force: logger.debug("Device %s repair budget exhausted; not retrying", device_id) @@ -716,6 +721,11 @@ def device_repair_due(device): # The operator removed this device on purpose. Never resurrect it -- # not on any attempt, and not under force. return False + if multi_attach.not_servable_here(device, device.node_id): + # Multi-attach: another node serves the LUN (or the owner is fenced + # from it). Repair rebuilds the stack on the owner, which must not + # hold one; the serving node's stack is the takeover's business. + return False schedule = constants.DEVICE_REPAIR_BACKOFF_SEC if device.repair_attempts >= len(schedule): return False diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index 73dfe8ef0f..2869d3aa0b 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -711,15 +711,13 @@ def unfence_node(device_id, node_id) -> bool: if dev.serving_node() == node_id: logger.error("Unfence of %s on %s refused: it is the serving node", node_id, device_id) return False - try: - if not array.plugin_for_cluster(cluster).unfence( - multi_attach.array_lun_ref(dev), multi_attach.array_host_ref(node)): - logger.error("Unfence of %s on LUN %s not acknowledged", node_id, dev.lun_wwn) - return False - except Exception as e: # noqa: BLE001 - logger.error("Unfence of %s on LUN %s failed: %s", node_id, dev.lun_wwn, e) - return False + # The record first, the array second. The attachment reconcile + # (array_controller.reconcile_lun_attachments, health check) detaches + # every host the record still lists as fenced: attaching first left a + # window in which it detached the LUN again (2026-11-19 FTT1 bed, kill of + # an owner's SPDK). Once the record is clear the reconcile attaches the + # node itself, so a failed array call below is retried there. def _mut(n): d = _device_on_owner(n, device_id) if d is None or node_id not in d.fenced_node_ids: @@ -729,4 +727,12 @@ def _mut(n): db.atomic_update(owner, _mut) _event(owner.cluster_id, dev, f"Node {node_id} unfenced from LUN {dev.lun_wwn} of device {device_id}") + try: + if not array.plugin_for_cluster(cluster).unfence( + multi_attach.array_lun_ref(dev), multi_attach.array_host_ref(node)): + logger.warning("Unfence of %s on LUN %s not acknowledged by the array; the " + "attachment reconcile presents it", node_id, dev.lun_wwn) + except Exception as e: # noqa: BLE001 + logger.warning("Array unfence of %s on LUN %s failed (%s); the attachment " + "reconcile presents it", node_id, dev.lun_wwn, e) return True diff --git a/simplyblock_core/services/device_monitor.py b/simplyblock_core/services/device_monitor.py index c0695401f4..89f17f39fc 100644 --- a/simplyblock_core/services/device_monitor.py +++ b/simplyblock_core/services/device_monitor.py @@ -1,6 +1,6 @@ import time -from simplyblock_core import constants, db_controller, utils +from simplyblock_core import constants, db_controller, multi_attach, utils from simplyblock_core.controllers import device_controller, tasks_controller from simplyblock_core.controllers.device_controller import CAUSE_LOCAL_FAILURE from simplyblock_core.models.cluster import Cluster @@ -46,7 +46,7 @@ def _check_aio_hung_io(node, rpc_client) -> list: into device failures (cf. constants NVME_TIMEOUT_US rationale).""" stalled = [] for dev in node.nvme_devices: - if dev.bdev_type != "aio" or dev.status != NVMeDevice.STATUS_ONLINE: + if dev.bdev_type != "aio" or dev.status != NVMeDevice.STATUS_ONLINE or _not_local(dev, node): _aio_progress.pop(dev.get_id(), None) continue try: @@ -82,6 +82,17 @@ def _check_aio_hung_io(node, rpc_client) -> list: return stalled +def _not_local(dev, node) -> bool: + """Multi-attach: the device's LUN is not served from ``node`` (a takeover + moved it, or ``node`` is fenced from it). The node holds no stack for it + and its host may not even see the LUN (the fence detached it), so neither + a missing disk nor a missing stack there says anything about the device. + 2026-11-19 FTT1 bed: after an SPDK kill and takeover, the fenced owner's + detached LUN was swept as hot-removed; device_remove of a 1+0 device + turned into read errors on every lvstore and a cluster suspension.""" + return multi_attach.not_servable_here(dev, node.get_id()) + + def _check_aio_device_presence(node) -> list: """Return the node's aio devices whose backing block device has been absent from the host inventory for AIO_DEVICE_ABSENT_POLLS consecutive @@ -89,7 +100,8 @@ def _check_aio_device_presence(node) -> list: aio_devs = [dev for dev in node.nvme_devices if dev.bdev_type == "aio" and dev.status in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_UNAVAILABLE, - NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]] + NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE] + and not _not_local(dev, node)] if not aio_devs: return [] try: diff --git a/tests/unit/test_lblk_watchdog.py b/tests/unit/test_lblk_watchdog.py index afe7d03017..b7ca035878 100644 --- a/tests/unit/test_lblk_watchdog.py +++ b/tests/unit/test_lblk_watchdog.py @@ -217,6 +217,64 @@ def test_no_aio_devices_no_inventory_call(self): node.client.assert_not_called() +def _ma_dev(serving="", fenced=(), node_id="node-1"): + d = _aio_dev() + d.multi_attach = True + d.node_id = node_id + d.serving_node_id = serving + d.fenced_node_ids = list(fenced) + return d + + +class TestMultiAttachNotLocal(WatchdogBase): + """2026-11-19 FTT1 bed: a multi-attach device whose LUN another node + serves (or whose owner is fenced from it) is not the owner's to sweep: + the fence detached the LUN from the owner's host, and a hot-removal + verdict turned into device_remove and a cluster suspension.""" + + def _absent(self, dev): + node = _node([dev]) + client = MagicMock() + client.get_blockdevices.return_value = ([{"name": "sdz", "serial": "ZZ"}], None) + node.client.return_value = client + return node + + def test_served_elsewhere_never_reported_absent(self): + dev = _ma_dev(serving="node-2") + node = self._absent(dev) + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS + 2): + self.assertEqual(_check_aio_device_presence(node), []) + + def test_fenced_owner_never_reported_absent(self): + dev = _ma_dev(fenced=["node-1"]) + node = self._absent(dev) + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS + 2): + self.assertEqual(_check_aio_device_presence(node), []) + + def test_served_here_still_swept(self): + dev = _ma_dev() + node = self._absent(dev) + for _ in range(constants.AIO_DEVICE_ABSENT_POLLS - 1): + self.assertEqual(_check_aio_device_presence(node), []) + self.assertEqual(_check_aio_device_presence(node), [dev]) + + def test_served_elsewhere_never_stalls(self): + dev = _ma_dev(serving="node-2") + node = _node([dev]) + rpc = _rpc_with_stats({dev.nvme_bdev: _stat(100, 5)}) + for _ in range(constants.AIO_HUNG_IO_STALL_POLLS + 2): + self.assertEqual(_check_aio_hung_io(node, rpc), []) + rpc.get_lvol_stats.assert_not_called() + + def test_repair_not_due_when_served_elsewhere(self): + from simplyblock_core.controllers import device_controller + dev = _ma_dev(serving="node-2") + dev.status = NVMeDevice.STATUS_UNAVAILABLE + self.assertFalse(device_controller.device_repair_due(dev)) + dev.serving_node_id = "" + self.assertTrue(device_controller.device_repair_due(dev)) + + class TestSweepActions(WatchdogBase): def _sweep(self, node, stalled=None, gone=None): diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index 9996827246..a49896fa04 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -844,3 +844,28 @@ def test_stale_stack_is_retired_first(self): self.assertEqual(lun_takeover.reintegrate_fenced_node(a), [DEV_ID]) self.assertTrue(a.spdk.ops("bdev_alceml_delete")) self.assertEqual(self.dev(db).fenced_node_ids, []) + + +class TestUnfenceOrder(_TakeoverCase): + """The fence record is cleared before the array call: the attachment + reconcile detaches any host the record still lists as fenced.""" + + def test_record_cleared_before_array_attach(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + seen = [] + orig = FakeArrayPlugin.unfence + + def _unfence(plugin, lun, host): + seen.append(list(self.dev(db).fenced_node_ids)) + return orig(plugin, lun, host) + with patch.object(FakeArrayPlugin, "unfence", _unfence): + self.assertTrue(lun_takeover.unfence_node(DEV_ID, "node-a")) + self.assertEqual(seen, [[]]) + + def test_array_failure_keeps_record_cleared(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + with patch.object(FakeArrayPlugin, "unfence", side_effect=RuntimeError("api")): + self.assertTrue(lun_takeover.unfence_node(DEV_ID, "node-a")) + self.assertEqual(self.dev(db).fenced_node_ids, []) From 13be6818a4f841686fa2483605645ddba10b04bd Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 19 Nov 2026 22:45:24 +0100 Subject: [PATCH 20/50] fix(multi-attach): grow a taken-over LUN on its serving node grow-devices refused every device whose LUN another node serves after a takeover ("grow it once its owner serves it again"). Nothing ever moves a LUN back to its owner, so after the first outage the cluster could not grow at all. Growth now rescans and reopens the alceml on the serving node, over the base bdev and path the takeover recorded. With --lun-factor the checks ran only after the LUNs were resized at the array: a refusal left the LUNs grown and the cluster not, and a re-run grew them again. Every device is now checked before the array is touched. A node that kept its base bdev over a LUN from before a growth (a former owner) still sees the old size; a takeover onto it failed the size check. The takeover now rescans the block device and the bdev before giving up. Co-Authored-By: Claude Opus 5.5 --- .../controllers/device_controller.py | 84 ++++++++++++++----- simplyblock_core/controllers/lun_takeover.py | 16 ++++ tests/unit/test_array_lifecycle.py | 25 ++++++ tests/unit/test_lblk_device_resize.py | 46 ++++++++++ tests/unit/test_lun_takeover.py | 38 ++++++++- 5 files changed, 189 insertions(+), 20 deletions(-) diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index 76714cbb50..4fa68e357e 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -2249,8 +2249,39 @@ def _growth_devices(db_controller, cluster_id): return pairs -def _check_resizable(cluster, dev, snode): - """Raise ValueError when ``dev`` cannot be grown online right now.""" +def _resizable_errors(db_controller, cluster, pairs) -> dict: + """``{device_id: reason}`` for every device of ``pairs`` that cannot be + grown online right now (_check_resizable on its serving node).""" + errors = {} + for node, dev in pairs: + try: + host = _growth_host(db_controller, node, dev)[0] + _check_resizable(cluster, dev, node, host) + except (ValueError, KeyError) as e: + errors[dev.get_id()] = str(e) + return errors + + +def _growth_host(db_controller, snode, dev): + """``(node, base_bdev, path)`` where the alceml of ``dev`` runs now: the + owner ``snode``, or for a multi-attach device whose LUN another node took + over (lun_takeover), that serving node, with the base bdev and block + device path the takeover recorded for it.""" + if not multi_attach.served_elsewhere(dev, snode.get_id()): + return snode, dev.nvme_bdev, dev.by_id_path or dev.device_path + host = db_controller.get_storage_node_by_id(dev.serving_node()) + p = (dev.node_paths or {}).get(host.get_id()) or {} + base = p.get("bdev") or utils.aio_bdev_name_for_serial(dev.serial_number) + return host, base, p.get("path") or dev.by_id_path or dev.device_path + + +def _check_resizable(cluster, dev, snode, host=None): + """Raise ValueError when ``dev`` (owned by ``snode``) cannot be grown + online right now. ``host`` is the node whose alceml serves it + (_growth_host), the owner by default: a LUN served elsewhere after a + takeover is grown on its serving node. The owner must still be online, + so growth never runs while a node of the cluster is out.""" + host = host or snode if cluster.device_mode != constants.DEVICE_MODE_LBLK or dev.bdev_type != "aio": raise ValueError("Online resizing is only supported for Linux block devices (lblk device mode)") if dev.status != NVMeDevice.STATUS_ONLINE: @@ -2263,18 +2294,20 @@ def _check_resizable(cluster, dev, snode): raise ValueError("Device runs with a testing or hang bdev under alceml, remove it first") if not dev.alceml_bdev or not dev.nvme_bdev: raise ValueError("Device has no alceml or AIO bdev recorded") - if multi_attach.served_elsewhere(dev, snode.get_id()): - # The alceml runs on the takeover node; this path grows the owner's. - raise ValueError(f"The device's LUN is served by node {dev.serving_node()} after a " - "takeover; grow it once its owner serves it again") + if host.get_id() != snode.get_id() and host.status != StorageNode.STATUS_ONLINE: + raise ValueError(f"The device's LUN is served by node {host.get_id()} after a takeover; " + f"that node must be online, current status: {host.status}") + if multi_attach.served_elsewhere(dev, host.get_id()): + raise ValueError(f"The device's LUN is served by node {dev.serving_node()}, not by " + f"{host.get_id()}") if tasks_controller.get_active_dev_restart_task(snode.cluster_id, dev.get_id()): raise ValueError("A restart task is active for the device") -def _rescan_size(snode, dev): +def _rescan_size(snode, dev, path=None): """Make the storage node's host re-read the size of the device's block device (SCSI or NVMe rescan) and return it in bytes.""" - path = dev.by_id_path or dev.device_path + path = path or dev.by_id_path or dev.device_path if not path: raise ValueError("No block device path known for the device") result, err = snode.client(timeout=60, retry=1).rescan_block_device(path) @@ -2315,11 +2348,13 @@ def _resize_failed(device_id, dev, msg, set_unavailable): return ValueError(msg) -def _reopen_alceml(snode, dev): +def _reopen_alceml(snode, dev, base_bdev=None): """Let alceml reopen its AIO bdev at the size the block device has now: resize_begin -> bdev_aio_rescan (bdev_uring_rescan for a uring device) - -> resize_end. Raises ValueError.""" + -> resize_end, on ``snode`` (the node serving the device) with its base + bdev ``base_bdev`` (default: the owner's). Raises ValueError.""" device_id = dev.get_id() + base_bdev = base_bdev or dev.nvme_bdev rpc_client = snode.rpc_client() io_engine = utils.lblk_io_engine_of_device(snode, dev) try: @@ -2332,7 +2367,7 @@ def _reopen_alceml(snode, dev): rescan_error = None try: - utils.rescan_lblk_bdev(rpc_client, dev.nvme_bdev, io_engine) + utils.rescan_lblk_bdev(rpc_client, base_bdev, io_engine) except RPCException as e: rescan_error = e @@ -2367,6 +2402,19 @@ def grow_cluster_devices(cluster_id, lun_factor=None): if lun_factor is None: return _grow_cluster_devices(cluster_id) from simplyblock_core.controllers import array_controller + # Check before the array is touched: a refusal after the LUNs grew would + # leave them grown and the cluster not, and a re-run with the factor + # would grow them again. + db_controller = DBController() + cluster = db_controller.get_cluster_by_id(cluster_id) + if cluster.device_mode == constants.DEVICE_MODE_LBLK: + errors = _resizable_errors(db_controller, cluster, _growth_devices(db_controller, cluster_id)) + if errors: + result = _growth_result(False, 1.0, [], errors, + "Not every data device can be grown; no LUN was grown at " + "the array") + result["luns"] = {} + return result lun_report = array_controller.grow_luns(cluster_id, lun_factor) if lun_report["errors"]: result = _growth_result(False, 1.0, [], dict(lun_report["errors"]), @@ -2404,21 +2452,18 @@ def _grow_cluster_devices(cluster_id): if not pairs: return _growth_result(False, 1.0, [], {}, "The cluster has no data devices") - errors = {} - for node, dev in pairs: - try: - _check_resizable(cluster, dev, node) - except ValueError as e: - errors[dev.get_id()] = str(e) + errors = _resizable_errors(db_controller, cluster, pairs) if errors: return _growth_result(False, 1.0, [], errors, "Not every data device can be grown; growth is uniform across all " "data devices, so nothing was grown") + hosts = {dev.get_id(): _growth_host(db_controller, node, dev) for node, dev in pairs} backend = {} for node, dev in pairs: + host, _, path = hosts[dev.get_id()] try: - backend[dev.get_id()] = _rescan_size(node, dev) + backend[dev.get_id()] = _rescan_size(host, dev, path) except ValueError as e: errors[dev.get_id()] = str(e) if errors: @@ -2456,8 +2501,9 @@ def _grow_cluster_devices(cluster_id): logger.info(f"Growing {len(pairs)} data devices of cluster {cluster_id} by factor {factor:.4f}") for node, dev in pairs: + host, base, _ = hosts[dev.get_id()] try: - _reopen_alceml(node, dev) + _reopen_alceml(host, dev, base) except ValueError as e: errors[dev.get_id()] = str(e) return _growth_result(False, factor, [], errors, diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index 2869d3aa0b..4c56913b65 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -385,6 +385,22 @@ def _export_on_standby(db, cluster, standby, dev, path, deadline) -> dict: if not info: raise TakeoverError(f"base bdev {base} not found on {sid} after create") size = int(info.get("block_size", 0)) * int(info.get("num_blocks", 0)) + if size < int(dev.size or 0) and ("base", base) not in created: + # A base bdev this node kept from before a LUN growth + # (device_controller.grow_cluster_devices) still has the old size: + # re-read the block device before giving up. + # The host's kernel may not have seen the growth either (the + # node was out while it ran). + try: + standby.client(timeout=_remaining(deadline, 30), retry=1).rescan_block_device(path) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: block rescan of %s on %s failed: %s", path, sid, e) + try: + utils.rescan_lblk_bdev(rpc, base, utils.lblk_io_engine_for({"path": path})) + info = rpc.bdev_get(base) or info + size = int(info.get("block_size", 0)) * int(info.get("num_blocks", 0)) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: rescan of %s on %s failed: %s", base, sid, e) if size < int(dev.size or 0): raise TakeoverError(f"base bdev {base} on {sid} is {size} bytes, the device {dev.size}") diff --git a/tests/unit/test_array_lifecycle.py b/tests/unit/test_array_lifecycle.py index cb40c707b0..de835e6983 100644 --- a/tests/unit/test_array_lifecycle.py +++ b/tests/unit/test_array_lifecycle.py @@ -583,7 +583,31 @@ def test_refusals_and_errors(self): with self.assertRaisesRegex(ValueError, "array"): array_controller.grow_luns(CID, 2) + def _prechecked(self, errors=None): + """Patch the pre-array check of grow_cluster_devices.""" + db = MagicMock() + db.get_cluster_by_id.return_value = MagicMock(device_mode=device_controller.constants.DEVICE_MODE_LBLK) + for target, attr, kw in ((device_controller, "DBController", {"return_value": db}), + (device_controller, "_growth_devices", {"return_value": []}), + (device_controller, "_resizable_errors", {"return_value": errors or {}})): + p = patch.object(target, attr, **kw) + p.start() + self.addCleanup(p.stop) + + def test_a_device_that_cannot_grow_leaves_the_array_alone(self): + # Checked before the array is touched: LUNs grown at the array with + # the cluster refusing would be grown again by a re-run. + self._prechecked({"d1": "Storage node must be online, current status: offline"}) + with patch.object(array_controller, "grow_luns") as grow_luns, patch.object(device_controller, "_grow_cluster_devices") as inner: + out = device_controller.grow_cluster_devices(CID, lun_factor=2) + grow_luns.assert_not_called() + inner.assert_not_called() + self.assertFalse(out["grown"]) + self.assertIn("no LUN was grown", out["reason"]) + self.assertEqual(out["luns"], {}) + def test_a_failed_lun_grows_nothing(self): + self._prechecked() # Growth is uniform (PR #1572): a LUN that did not grow at the array # would cap the factor at 1, so nothing on the cluster is grown. with patch.object(array_controller, "grow_luns", @@ -595,6 +619,7 @@ def test_a_failed_lun_grows_nothing(self): self.assertEqual(out["luns"], {}) def test_grown_luns_then_uniform_growth(self): + self._prechecked() luns = {"d1": {"lun_id": "vol-1", "old_size": 1, "new_size": 2}} with patch.object(array_controller, "grow_luns", return_value={"luns": luns, "errors": {}}), patch.object(device_controller, "_grow_cluster_devices", return_value={"grown": True, "factor": 2.0, "devices": [], "errors": {}, diff --git a/tests/unit/test_lblk_device_resize.py b/tests/unit/test_lblk_device_resize.py index acdfea130d..2b7accc81c 100644 --- a/tests/unit/test_lblk_device_resize.py +++ b/tests/unit/test_lblk_device_resize.py @@ -317,6 +317,52 @@ def test_begin_failure_reopens(self): h.set_unavail.assert_not_called() +class TestGrowAfterTakeover(unittest.TestCase): + """A multi-attach LUN another node took over is grown where its alceml + runs: the serving node, over the base bdev and path the takeover + recorded. The owner (back as a standby) is not touched.""" + + def _harness(self, host_status=StorageNode.STATUS_ONLINE): + h = _GrowHarness([150, 150]) + moved = h.devs[1] + moved.multi_attach = True + moved.serving_node_id = "node-2" + moved.node_paths = {"node-2": {"path": "/dev/disk/by-id/lun-on-2", "bdev": "aio_on_2"}} + host = _node([], status=host_status) + host.get_id.return_value = "node-2" + host.rpc = MagicMock() + host.rpc_client.return_value = host.rpc + host.agent = MagicMock() + host.agent.rescan_block_device.return_value = ({"size": 150 * _GIB}, None) + host.client.return_value = host.agent + h.db.get_storage_node_by_id.side_effect = lambda nid: host if nid == "node-2" else h.node + return h, moved, host + + def test_grown_on_the_serving_node(self): + h, moved, host = self._harness() + with patch.object(device_controller.utils, "lblk_io_engine_of_device", + return_value=constants.LBLK_IO_ENGINE_AIO): + res = h.grow() + self.assertTrue(res["grown"], res) + self.assertEqual(h.sizes(), [150, 150]) + host.agent.rescan_block_device.assert_called_once_with("/dev/disk/by-id/lun-on-2") + host.rpc.alceml_bdev_backend_resize_begin.assert_called_once_with( + "alceml_dev-1", request_timeout=device_controller.RESIZE_RPC_TIMEOUT_SEC) + host.rpc.bdev_aio_rescan.assert_called_once_with("aio_on_2") + owner_calls = [c for c in h.rpc.mock_calls if "alceml_dev-1" in str(c) or "aio_dev-1" in str(c)] + self.assertEqual(owner_calls, []) + self.assertNotIn(moved.by_id_path, [c.args[0] for c in h.client.rescan_block_device.call_args_list]) + + def test_serving_node_offline_grows_nothing(self): + h, _, host = self._harness(host_status=StorageNode.STATUS_OFFLINE) + res = h.grow() + self.assertFalse(res["grown"]) + self.assertIn("served by node node-2", res["errors"]["dev-1"]) + self.assertEqual(h.sizes(), [100, 100]) + host.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + h.rpc.alceml_bdev_backend_resize_begin.assert_not_called() + + class TestUnusedCapacity(unittest.TestCase): """The capacity statistics leave out the part of a block device beyond the device size: alceml spans the whole block device.""" diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index a49896fa04..315a5c7600 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -16,7 +16,7 @@ import unittest from unittest.mock import MagicMock, patch -from simplyblock_core import array, constants, multi_attach, storage_node_ops +from simplyblock_core import array, constants, multi_attach, storage_node_ops, utils from simplyblock_core.array.fake import FakeArrayPlugin from simplyblock_core.controllers import device_controller, lun_takeover from simplyblock_core.models.cluster import Cluster @@ -869,3 +869,39 @@ def test_array_failure_keeps_record_cleared(self): with patch.object(FakeArrayPlugin, "unfence", side_effect=RuntimeError("api")): self.assertTrue(lun_takeover.unfence_node(DEV_ID, "node-a")) self.assertEqual(self.dev(db).fenced_node_ids, []) + + +class TestTakeoverAfterGrowth(_TakeoverCase): + """A node that kept the base bdev over a LUN from before a growth + (device_controller.grow_cluster_devices) still has it at the old size; + the takeover re-reads the block device instead of failing.""" + + def _grown_bed(self): + db, dev = _bed() + dev.size = 2 * SIZE + for nid in ("node-b", "node-c", "node-d"): + db.nodes[nid].agent.lun_info.return_value = ([_lun_entry(size=2 * SIZE)], None) + b = db.nodes["node-b"] + base = utils.aio_bdev_name_for_serial(dev.serial_number) + b.spdk.bdevs[base] = {"name": base, "block_size": 4096, "num_blocks": SIZE // 4096} + return db, b, base + + def test_stale_base_bdev_is_rescanned(self): + db, b, base = self._grown_bed() + + def _rescan(name): + b.spdk.calls.append(("bdev_aio_rescan", (name,), {})) + b.spdk.bdevs[name]["num_blocks"] = 2 * SIZE // 4096 + return True + b.spdk.bdev_aio_rescan = _rescan + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(b.spdk.ops("bdev_aio_rescan"), [("bdev_aio_rescan", (base,), {})]) + b.agent.rescan_block_device.assert_called_once_with(BY_ID) + self.assertEqual(b.spdk.ops("bdev_aio_create"), []) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + def test_base_bdev_still_small_after_rescan_fails_on_that_node(self): + db, b, base = self._grown_bed() + b.spdk.bdev_aio_rescan = lambda name: True + self.assertNotEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) From cb415d932545185ccfdd822da1ee3f6c0248e4a6 Mon Sep 17 00:00:00 2001 From: michael Date: Thu, 19 Nov 2026 22:47:40 +0100 Subject: [PATCH 21/50] fix(multi-attach): read a taken-over LUN's stats on its serving node The capacity and stats collector asked the owner for the capacity and I/O stats of every device. After a takeover the owner has no alceml for it, the call failed and the device was skipped, so the cluster capacity dropped by one device per taken-over LUN (and grown capacity was never seen for it). Read them on the serving node, over the base bdev the takeover recorded; skip the device only while that node is not online. Co-Authored-By: Claude Opus 5.5 --- .../services/capacity_and_stats_collector.py | 29 ++++++++++++++-- tests/unit/test_lblk_device_resize.py | 33 +++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/simplyblock_core/services/capacity_and_stats_collector.py b/simplyblock_core/services/capacity_and_stats_collector.py index bb1fbc5e58..6f1b49f27d 100644 --- a/simplyblock_core/services/capacity_and_stats_collector.py +++ b/simplyblock_core/services/capacity_and_stats_collector.py @@ -1,7 +1,7 @@ import statistics import time -from simplyblock_core import constants, db_controller, utils +from simplyblock_core import constants, db_controller, multi_attach, utils from simplyblock_core.controllers import device_events from simplyblock_core.models.nvme_device import NVMeDevice from simplyblock_core.models.stats import ( @@ -364,6 +364,22 @@ def add_cluster_stats(cl, records): +def device_stats_source(device, owner, owner_rpc, nodes_by_id): + """``(rpc_client, base_bdev)`` to read the capacity and I/O stats of + ``device`` from: its owner, or for a multi-attach device whose LUN + another node took over (lun_takeover), the serving node, where its + alceml runs now, with the base bdev the takeover recorded there. None + while that node is not online.""" + if not multi_attach.served_elsewhere(device, owner.get_id()): + return owner_rpc, device.nvme_bdev + host = nodes_by_id.get(device.serving_node()) + if host is None or host.status != StorageNode.STATUS_ONLINE: + return None + path = (device.node_paths or {}).get(host.get_id()) or {} + base = path.get("bdev") or utils.aio_bdev_name_for_serial(device.serial_number) + return host.rpc_client(timeout=5, retry=2), base + + # get DB controller db = db_controller.DBController() @@ -384,6 +400,7 @@ def main(): logger.error(f"Cluster has no storage nodes: {cl.get_id()}") all_lvols = db.get_mini_lvols() + nodes_by_id = {n.get_id(): n for n in snodes} node_records = [] cluster_device_records = [] for node in snodes: @@ -403,13 +420,19 @@ def main(): if device.status not in [NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE]: logger.info(f"Device is skipped: {device.get_id()} status: {device.status}") continue + source = device_stats_source(device, node, rpc_client, nodes_by_id) + if source is None: + logger.info(f"Device is skipped: {device.get_id()} its serving node " + f"{device.serving_node()} is not online") + continue + dev_rpc, base_bdev = source try: - capacity_dict = rpc_client.alceml_get_capacity(device.alceml_name) + capacity_dict = dev_rpc.alceml_get_capacity(device.alceml_name) except Exception as e: logger.error(e) continue try: - ret = rpc_client.get_lvol_stats(device.nvme_bdev) + ret = dev_rpc.get_lvol_stats(base_bdev) except RPCException as e: logger.error("Failed to get iostat for device %s: %s", device.get_id(), e) continue diff --git a/tests/unit/test_lblk_device_resize.py b/tests/unit/test_lblk_device_resize.py index 2b7accc81c..7de016121a 100644 --- a/tests/unit/test_lblk_device_resize.py +++ b/tests/unit/test_lblk_device_resize.py @@ -494,3 +494,36 @@ def test_virtio_without_rescan_reads_size(self): if __name__ == "__main__": unittest.main() + + +class TestStatsSourceAfterTakeover(unittest.TestCase): + """Capacity and I/O stats of a taken-over LUN come from the node whose + alceml serves it; read on the owner they were missing, and the cluster + capacity lost one device per takeover.""" + + def setUp(self): + from simplyblock_core.services import capacity_and_stats_collector as col + self.col = col + self.owner = _node([]) + self.owner_rpc = MagicMock() + self.host = _node([]) + self.host.get_id.return_value = "node-2" + self.host_rpc = MagicMock() + self.host.rpc_client.return_value = self.host_rpc + self.nodes = {"node-1": self.owner, "node-2": self.host} + self.dev = _aio_dev("dev-1") + + def test_owner_serves(self): + self.assertEqual(self.col.device_stats_source(self.dev, self.owner, self.owner_rpc, self.nodes), + (self.owner_rpc, "aio_dev-1")) + + def test_taken_over_reads_the_serving_node(self): + self.dev.multi_attach, self.dev.serving_node_id = True, "node-2" + self.dev.node_paths = {"node-2": {"path": "/dev/x", "bdev": "aio_on_2"}} + self.assertEqual(self.col.device_stats_source(self.dev, self.owner, self.owner_rpc, self.nodes), + (self.host_rpc, "aio_on_2")) + + def test_serving_node_offline_is_skipped(self): + self.dev.multi_attach, self.dev.serving_node_id = True, "node-2" + self.host.status = StorageNode.STATUS_OFFLINE + self.assertIsNone(self.col.device_stats_source(self.dev, self.owner, self.owner_rpc, self.nodes)) From 251c38f85af403d40f8aebce1c7419af2cce514b Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 10:11:47 +0100 Subject: [PATCH 22/50] feat(multi-attach): take a lost node's LUNs over within seconds With 1+0 every volume waits on a lost node's LUN until a standby serves it. On the FTT1 bed the stall was 9-10 s for a killed SPDK and 24-35 s for a reboot, hard reset or 20 s network cut: the takeover started from the regular monitor's UNREACHABLE/OFFLINE verdicts plus a fixed 15 s wait. - services/lun_watchdog (started by the storage node monitor): probes every node serving a LUN every 0.5 s (spdk_get_version, 0.8 s timeout; the node agent while it fails). Lost at once when the agent reports SPDK not running, after 2 s when neither answers; never while SPDK runs. A node answering again within the window is not taken over. No takeover while more nodes fail than the cluster tolerates or no other node answered recently (this monitor cut off). - The 15 s confirmation (MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC) is gone: the regular monitor's UNREACHABLE starts the takeover at once (backstop). - Takeover: the array connection is cached and warmed (connecting cost ~1 s per fence) with one retry on a fresh one; EBS fence polls the detach every 0.2 s instead of 2 s; the standby is chosen (read-only LUN probe) while the fence runs, nothing is built before its ack; the new path is added on all peers in parallel. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/array/__init__.py | 49 +++ simplyblock_core/array/aws_ebs.py | 21 +- simplyblock_core/constants.py | 29 +- simplyblock_core/controllers/lun_takeover.py | 78 +++-- simplyblock_core/services/lun_watchdog.py | 324 ++++++++++++++++++ .../services/storage_node_monitor.py | 23 +- tests/unit/test_array_lifecycle.py | 13 + tests/unit/test_lun_takeover.py | 93 +++-- tests/unit/test_lun_watchdog.py | 264 ++++++++++++++ 9 files changed, 813 insertions(+), 81 deletions(-) create mode 100644 simplyblock_core/services/lun_watchdog.py create mode 100644 tests/unit/test_lun_watchdog.py diff --git a/simplyblock_core/array/__init__.py b/simplyblock_core/array/__init__.py index 3ff129b19e..cff580e013 100644 --- a/simplyblock_core/array/__init__.py +++ b/simplyblock_core/array/__init__.py @@ -24,8 +24,10 @@ from __future__ import annotations import abc +import json import logging import threading +import time from importlib import metadata logger = logging.getLogger() @@ -305,3 +307,50 @@ def plugin_for_cluster(cluster) -> ArrayPlugin: credentials=array_credentials(cluster), cluster_id=cluster.get_id(), ) + + +# cluster id -> (configuration key, monotonic time connected, plugin) +_connected: dict[str, tuple] = {} +_connected_lock = threading.Lock() + + +def _config_key(cluster) -> tuple: + return (getattr(cluster, "array_plugin", "") or "", + getattr(cluster, "array_endpoint", "") or "", + json.dumps(getattr(cluster, "array_options", None) or {}, sort_keys=True, default=str), + getattr(cluster, "array_id", "") or "", + getattr(cluster, "array_has_credentials", False) is True) + + +def cached_plugin_for_cluster(cluster, max_age: float | None = None) -> ArrayPlugin: + """:func:`plugin_for_cluster`, connected once and reused for + ``constants.ARRAY_PLUGIN_CACHE_SEC`` while the cluster's array + configuration is unchanged. For the LUN takeover's fence, which every + volume of a 1+0 cluster waits on: connecting validates the credentials + first (about a second on EBS). A caller whose call fails on the cached + instance drops it with :func:`forget_plugin` and connects again. + ``max_age`` (seconds) connects anew when the cached one is older (a + refresh ahead of the expiry, so a fence never waits for it).""" + from simplyblock_core import constants + cid = cluster.get_id() + key = _config_key(cluster) + # The class too: a plugin registered anew under the same name (tests). + key = (plugin_class(key[0]) if key[0] else None,) + key + with _connected_lock: + hit = _connected.get(cid) + limit = constants.ARRAY_PLUGIN_CACHE_SEC if max_age is None else max_age + if hit and hit[0] == key and time.monotonic() - hit[1] < limit: + return hit[2] + plugin = plugin_for_cluster(cluster) + with _connected_lock: + _connected[cid] = (key, time.monotonic(), plugin) + return plugin + + +def forget_plugin(cluster_id: str | None = None) -> None: + """Drop the cached connection of one cluster (all with None).""" + with _connected_lock: + if cluster_id is None: + _connected.clear() + else: + _connected.pop(cluster_id, None) diff --git a/simplyblock_core/array/aws_ebs.py b/simplyblock_core/array/aws_ebs.py index e68888f145..70d42491cb 100644 --- a/simplyblock_core/array/aws_ebs.py +++ b/simplyblock_core/array/aws_ebs.py @@ -33,7 +33,8 @@ Options: ``az`` (required), ``region`` (default: ``az`` without its letter), ``volume_type`` (``io2`` default, or ``io1``), ``iops`` (default 3000), ``encrypted`` (default true), ``kms_key_id``, ``tags`` (dict, added to every -volume), ``poll_interval`` (s, default 2), ``timeout`` (s, default 300). +volume), ``poll_interval`` (s, default 2), ``fence_poll_interval`` (s, default +0.2: a fence holds the volumes), ``timeout`` (s, default 300). ``endpoint`` (the registration's endpoint) is an EC2 endpoint URL override (VPC endpoint), empty for the regional default. """ @@ -130,6 +131,11 @@ def region(self) -> str: def _poll(self) -> float: return float(self.options.get("poll_interval", 2)) + @property + def _fence_poll(self) -> float: + # A fence holds every volume of a 1+0 cluster: poll its detach fast. + return float(self.options.get("fence_poll_interval", 0.2)) + @property def _timeout(self) -> float: return float(self.options.get("timeout", 300)) @@ -196,7 +202,7 @@ def _summary(vol) -> dict: "state": vol.get("State", ""), } - def _wait(self, what, check): + def _wait(self, what, check, poll=None): """Poll ``check()`` until it returns a truthy value or the timeout.""" deadline = time.monotonic() + self._timeout while True: @@ -205,7 +211,7 @@ def _wait(self, what, check): return result if time.monotonic() >= deadline: raise ArrayError(f"aws-ebs: timed out waiting for {what}") - self._sleep(self._poll) + self._sleep(self._poll if poll is None else poll) def _free_device_name(self, instance_id) -> str: try: @@ -333,7 +339,12 @@ def _attached(): return a is not None and a.get("State") == "attached" self._wait(f"volume {vid} to attach to {host_id}", _attached) - def detach(self, lun_id, host_id, force=False): + def fence(self, lun_id, host_id): + """A forced detach, polled every ``fence_poll_interval`` (0.2 s).""" + self.detach(lun_id, host_id, force=True, poll=self._fence_poll) + return True + + def detach(self, lun_id, host_id, force=False, poll=None): vol = self._owned(lun_id) vid = vol["VolumeId"] att = self._attachment(vol, host_id) @@ -351,7 +362,7 @@ def _detached(): v = self._describe(vid) a = self._attachment(v or {}, host_id) return a is None or a.get("State") == "detached" - self._wait(f"volume {vid} to detach from {host_id}", _detached) + self._wait(f"volume {vid} to detach from {host_id}", _detached, poll) def resize(self, lun_id, size_bytes): vol = self._owned(lun_id) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 4dee4eab8b..c1b0c121bd 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -1329,9 +1329,10 @@ def get_config_var(name, default=None): # path is available (per-controller ctrlr_loss_timeout_sec), so a takeover # that adds the LUN's new export as a path within the window resumes I/O # instead of failing it. The cluster-wide bdev_nvme defaults (1 s) stay. -# It must cover: detecting the owner unreachable (~20 s), the -# MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC confirmation, and the takeover up to -# its MULTI_ATTACH_TAKEOVER_DEADLINE_SEC. +# It must cover the slowest takeover: the regular monitor's detection of the +# node (~20 s) when the fast probe (MULTI_ATTACH_PROBE_*) did not trigger, and +# the takeover up to its MULTI_ATTACH_TAKEOVER_DEADLINE_SEC. The I/O stall of a +# takeover started by the fast probe is a few seconds. MULTI_ATTACH_HOLD_SEC = 150 MULTI_ATTACH_RECONNECT_DELAY_SEC = 2 MULTI_ATTACH_FAST_IO_FAIL_SEC = MULTI_ATTACH_HOLD_SEC @@ -1352,13 +1353,21 @@ def get_config_var(name, default=None): # peers' hold, or their I/O fails before the new path exists; it gives up at # this deadline (seconds from its start) and the device goes unavailable. MULTI_ATTACH_TAKEOVER_DEADLINE_SEC = 90 -# The takeover of a node's LUNs starts once the node has been UNREACHABLE -# this long, not when it is declared OFFLINE: the OFFLINE verdict waits for -# every online peer to see the node's journal disconnected, which took about -# six minutes on the multi-attach bed, far past the peers' hold. The fence -# comes first, so a takeover of a node that is alive behind a management -# network fault is safe; it costs a short I/O pause on that LUN. -MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC = 15 +# Fast loss detection of a node serving LUNs (services/lun_watchdog). With +# 1+0 every volume waits on a lost node's LUN until its takeover, so the +# takeover starts from this probe, not from the node's status: the OFFLINE +# verdict took minutes on the multi-attach bed. The node's SPDK RPC is probed +# every PROBE_INTERVAL with PROBE_TIMEOUT, together with its node agent. A +# node is lost when its agent reports SPDK not running (at once), or when +# neither SPDK nor the agent has answered for LOST_AFTER. The fence comes +# first, so a takeover of a node that is alive behind a management network +# fault is safe; it costs a short I/O pause on its LUNs. +MULTI_ATTACH_PROBE_INTERVAL_SEC = 0.5 +MULTI_ATTACH_PROBE_TIMEOUT_SEC = 0.8 +MULTI_ATTACH_LOST_AFTER_SEC = 2.0 +# The array connection a fence uses is kept this long (a new one validates +# the credentials first: about a second on EBS). +ARRAY_PLUGIN_CACHE_SEC = 600 # Lease of a running takeover (one per LUN, in FDB): a holder older than this # is presumed dead. MULTI_ATTACH_TAKEOVER_LOCK_TTL_SEC = 2 * MULTI_ATTACH_TAKEOVER_DEADLINE_SEC diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index 4c56913b65..454b5341a8 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -10,7 +10,8 @@ 2. picks a standby: the owner's LVS secondary, else its tertiary, else the owner itself; ONLINE and not fenced from this LUN; 3. verifies through the standby's node agent that it sees the LUN and that - block 0 carries the alceml signature (read-only); + block 0 carries the alceml signature (read-only); 2 and 3 run while the + fence is applied, nothing is built on the standby before its ack; 4. on the standby builds the base bdev, the alceml OPEN-ONLY (``pba_init_mode=1``; 2/3 would format the LUN and destroy its data), the PT bdev and the device subsystem under the same NQN, with the standby's @@ -18,7 +19,8 @@ 5. records the standby as the serving node (atomically; the device stays ONLINE, no unavailable broadcast, no cluster-map change); 6. adds the standby's export as a path of ``remote_alceml_`` on every - ONLINE node (the standby too: its loopback), where the held I/O resumes; + ONLINE node (the standby too: its loopback), all in parallel, where the + held I/O resumes; 7. detaches the failed node's path from those controllers so it never reconnects. @@ -44,6 +46,7 @@ import threading import time import uuid as uuid_mod +from concurrent.futures import ThreadPoolExecutor from simplyblock_core import array, constants, multi_attach, utils from simplyblock_core.controllers import events_controller as ec @@ -217,14 +220,21 @@ def _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) old_ips = [ip.strip() for ip in (dev.nvmf_ip or "").split(",") if ip.strip()] old_port = dev.nvmf_port + # 2 + 3 run while the fence is applied: choosing the standby only reads + # (the node agent's LUN inventory), and nothing is built on it before + # the array has acknowledged the fence. Every volume of a 1+0 cluster + # waits on this LUN. + pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix=f"standby-{device_id[:8]}") + choice = pool.submit(_choose_standby, owner, dev, failed_node_id, nodes_by_id, deadline) + pool.shutdown(wait=False) try: - # 1. Fence -- before anything else. + # 1. Fence -- before anything else is changed. _fence(cluster, dev, failed) _record_fenced(db, owner, device_id, failed_node_id) _check_deadline(deadline, "choosing a standby") # 2 + 3. A standby that can read the LUN. - standby, path, why = _choose_standby(owner, dev, failed_node_id, nodes_by_id, deadline) + standby, path, why = choice.result(timeout=max(1.0, deadline - time.monotonic())) if standby is None: msg = (f"LUN {dev.lun_wwn} of device {device_id} not taken over from node " f"{failed_node_id}: no standby can read it ({why}); holding I/O until " @@ -283,18 +293,27 @@ def _fence(cluster, dev, failed): lun = multi_attach.array_lun_ref(dev) if not lun: raise TakeoverError("the device has no LUN WWN to fence") - try: - plugin = array.plugin_for_cluster(cluster) - except array.ArrayNotConfigured: - raise TakeoverError( - "no array plugin is configured (cluster array_plugin); a LUN is never " - "taken over without fencing its serving node") - except array.ArrayError as e: - raise TakeoverError(f"array plugin unavailable: {e}") - try: - ok = plugin.fence(lun, host) - except Exception as e: # noqa: BLE001 - raise TakeoverError(f"array fence of {host} on LUN {lun} failed: {e}") + ok = None + for fresh in (False, True): + if fresh: + # The cached connection may have gone stale (expired session). + array.forget_plugin(cluster.get_id()) + try: + plugin = array.cached_plugin_for_cluster(cluster) + except array.ArrayNotConfigured: + raise TakeoverError( + "no array plugin is configured (cluster array_plugin); a LUN is never " + "taken over without fencing its serving node") + except array.ArrayError as e: + raise TakeoverError(f"array plugin unavailable: {e}") + try: + ok = plugin.fence(lun, host) + break + except Exception as e: # noqa: BLE001 + if fresh: + raise TakeoverError(f"array fence of {host} on LUN {lun} failed: {e}") + logger.warning("LUN %s: fence of %s failed (%s); again on a new array connection", + lun, host, e) if not ok: raise TakeoverError(f"array did not acknowledge the fence of {host} on LUN {lun}") logger.info("LUN %s: node %s (%s) fenced at the array", lun, failed.get_id(), host) @@ -534,23 +553,23 @@ def _failed_paths(failed, old_ips, old_port): def _switch_paths(db, cluster_id, dev, standby, failed, failed_paths) -> list: """Add the standby's export as a path of ``remote_alceml_`` on every - ONLINE node, then detach the failed node's path there. Returns the ids - of the nodes that have the new path.""" + ONLINE node, then detach the failed node's path there; on all nodes in + parallel. Returns the ids of the nodes that have the new path.""" name = f"remote_{dev.alceml_bdev}" trtype = _transport(standby) or "TCP" new_ips = [ip.strip() for ip in (dev.nvmf_ip or "").split(",") if ip.strip()] dead_ips, dead_port, dead_trtype = failed_paths dead_ips = [ip for ip in dead_ips if ip not in new_ips] failed_id = failed.get_id() if failed is not None else None - added = [] - for node in db.get_storage_nodes_by_cluster_id(cluster_id): - if node.status != StorageNode.STATUS_ONLINE or node.get_id() == failed_id: - continue + nodes = [n for n in db.get_storage_nodes_by_cluster_id(cluster_id) + if n.status == StorageNode.STATUS_ONLINE and n.get_id() != failed_id] + + def _one(node): try: rpc = node.rpc_client(timeout=5, retry=1) except Exception as e: # noqa: BLE001 logger.warning("LUN takeover: no RPC client for %s: %s", node.get_id(), e) - continue + return False ok = False for ip in new_ips: try: @@ -564,8 +583,7 @@ def _switch_paths(db, cluster_id, dev, standby, failed, failed_paths) -> list: if not ok: logger.error("LUN takeover: %s on %s has no path to the new export; the health " "check reconnects it from the record", name, node.get_id()) - continue - added.append(node.get_id()) + return False try: rpc.bdev_nvme_set_multipath_policy(f"{name}n1", "active_passive") except Exception as e: # noqa: BLE001 @@ -579,7 +597,15 @@ def _switch_paths(db, cluster_id, dev, standby, failed, failed_paths) -> list: # Usually: the path is already gone (the hold destructed it). logger.info("LUN takeover: old path %s of %s on %s not detached: %s", ip, name, node.get_id(), e) - return added + return True + + if not nodes: + return [] + # All nodes at once: the held I/O of each resumes when its path is added. + with ThreadPoolExecutor(max_workers=min(16, len(nodes)), + thread_name_prefix=f"paths-{dev.get_id()[:8]}") as pool: + results = list(pool.map(_one, nodes)) + return [n.get_id() for n, ok in zip(nodes, results) if ok] def retire_stale_local_stack(node) -> list: diff --git a/simplyblock_core/services/lun_watchdog.py b/simplyblock_core/services/lun_watchdog.py new file mode 100644 index 0000000000..3ba5d97449 --- /dev/null +++ b/simplyblock_core/services/lun_watchdog.py @@ -0,0 +1,324 @@ +"""Multi-attach clusters: fast loss detection of the nodes serving LUNs. + +With 1+0 erasure coding every volume has data on every device, so all of +them wait on a lost node's LUN until a standby has taken it over +(controllers/lun_takeover). The regular monitor declares a node lost only +after its ping/API checks have failed: 1 s for a killed SPDK, 10-20 s for a +host that is gone, and the OFFLINE verdict takes far longer. That is the +volumes' I/O stall. + +This watchdog probes every node that serves a LUN every +``MULTI_ATTACH_PROBE_INTERVAL_SEC``: a ``spdk_get_version`` RPC with +``MULTI_ATTACH_PROBE_TIMEOUT_SEC``. While that fails it also asks the node +agent whether SPDK runs. The node is lost + +* at once when the agent reports SPDK not running (killed or crashed); +* when neither SPDK nor the agent has answered for + ``MULTI_ATTACH_LOST_AFTER_SEC`` (host down or cut off); + +and never while the agent reports SPDK running: a slow RPC is the regular +monitor's case. A node that answers again within the window is not taken +over at all. + +The takeover of a lost node's LUNs starts straight away, each LUN in its own +thread (the monitor's ``_start_lun_takeover``, idempotent). It fences the +node at the array before anything else, so a node that is alive behind a +management network fault loses nothing; its LUNs pause for the takeover. + +Not when this process is the one cut off: no takeover while more nodes of a +cluster fail their probes than it tolerates (``max_fault_tolerance``), or +while no other node of it has answered recently. +""" + +from __future__ import annotations + +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +from simplyblock_core import array, constants, multi_attach, utils +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + +logger = utils.get_logger(__name__) + +#: Node states the probe watches. A node in a restart or shutdown is owned +#: by that flow; an OFFLINE one was handled when it went offline. +WATCH_STATUSES = (StorageNode.STATUS_ONLINE, StorageNode.STATUS_UNREACHABLE, + StorageNode.STATUS_DOWN, StorageNode.STATUS_SCHEDULABLE) + +#: Device states whose LUN a lost node takes down with it. +SERVING_STATES = (NVMeDevice.STATUS_ONLINE, NVMeDevice.STATUS_READONLY, + NVMeDevice.STATUS_CANNOT_ALLOCATE) + +#: Probe answers. +OK = "ok" # SPDK answered +SPDK_DOWN = "spdk_down" # SPDK did not; the agent says it is not running +SPDK_UP = "spdk_up" # SPDK did not; the agent says it runs (slow RPC) +NO_ANSWER = "no_answer" # neither answered + +#: Re-trigger of a node still lost (its takeover threads are idempotent). +REFIRE_SEC = 5.0 +#: How often the set of watched nodes is re-read. +REFRESH_SEC = 1.0 + + +def _rpc_answers(node, timeout) -> bool: + try: + node.rpc_client(timeout=timeout, retry=0).get_version() + return True + except Exception: # noqa: BLE001 + return False + + +def _agent_says(node, timeout) -> str: + try: + is_up, _ = node.client(timeout=timeout, retry=0, connect_retry=0).spdk_process_is_up( + node.rpc_port, node.cluster_id) + except Exception: # noqa: BLE001 + return NO_ANSWER + if is_up is True: + return SPDK_UP + if is_up is False: + return SPDK_DOWN + return NO_ANSWER + + +def probe(node, failing=False, timeout=None) -> str: + """One probe of ``node``. While it is already ``failing`` the RPC and the + agent are asked at the same time, else the agent only after the RPC + failed.""" + timeout = timeout or constants.MULTI_ATTACH_PROBE_TIMEOUT_SEC + if not failing: + if _rpc_answers(node, timeout): + return OK + return _agent_says(node, timeout) + with ThreadPoolExecutor(max_workers=2) as pool: + rpc = pool.submit(_rpc_answers, node, timeout) + agent = pool.submit(_agent_says, node, timeout) + if rpc.result(): + return OK + return agent.result() + + +class _Track: + """Probe history of one node.""" + + __slots__ = ("answer", "fired_at", "first_fail", "last_ok") + + def __init__(self, now): + self.first_fail = None + self.last_ok = now + self.fired_at = None + self.answer = OK + + +class Watchdog: + """The decisions, apart from the threads that probe (testable).""" + + def __init__(self, start_takeover, db=None): + self.start_takeover = start_takeover + self.db = db + self.lock = threading.Lock() + self.tracks: dict[str, _Track] = {} + self.cluster_of: dict[str, str] = {} + self.warmed: dict[str, float] = {} + + def _db(self): + if self.db is None: + self.db = DBController() + return self.db + + # ----------------------------------------------------------- watched + + def watched(self) -> dict: + """``{node_id: node}`` of the nodes to probe now: in a multi-attach + cluster that is not in activation, in a WATCH_STATUSES state, serving + at least one LUN in a SERVING_STATES state.""" + out = {} + db = self._db() + for cluster in db.get_clusters(): + if not multi_attach.is_multi_attach(cluster): + continue + if cluster.status == Cluster.STATUS_IN_ACTIVATION: + continue + self._warm(cluster) + nodes = db.get_storage_nodes_by_cluster_id(cluster.get_id()) + for node in nodes: + if node.status not in WATCH_STATUSES: + continue + if any(dev.status in SERVING_STATES + for _o, dev in multi_attach.devices_served_by(nodes, node.get_id())): + out[node.get_id()] = node + return out + + def _warm(self, cluster): + """Keep the cluster's array connection ready for a fence (connecting + costs about a second), refreshed at half its cache time.""" + cid = cluster.get_id() + now = time.monotonic() + half = constants.ARRAY_PLUGIN_CACHE_SEC / 2 + if not getattr(cluster, "array_plugin", "") or now - self.warmed.get(cid, -1e9) < half: + return + self.warmed[cid] = now + + def _connect(): + try: + array.cached_plugin_for_cluster(cluster, max_age=half) + except Exception as e: # noqa: BLE001 + logger.warning("LUN watchdog: array connection of %s not made: %s", cid, e) + self.warmed[cid] = time.monotonic() - half + 30 # again in 30 s + threading.Thread(target=_connect, name=f"array-warm-{cid[:8]}", daemon=True).start() + + # ---------------------------------------------------------- decisions + + def record(self, node, answer, now) -> str | None: + """Note a probe answer of ``node`` at ``now``; the reason the node is + lost, or None.""" + nid = node.get_id() + with self.lock: + self.cluster_of[nid] = node.cluster_id + t = self.tracks.setdefault(nid, _Track(now)) + t.answer = answer + if answer == OK: + if t.first_fail is not None: + logger.info("LUN watchdog: %s answers again after %.1fs", + nid, now - t.first_fail) + t.first_fail, t.fired_at, t.last_ok = None, None, now + return None + if t.first_fail is None: + t.first_fail = now + if answer == SPDK_UP: + return None + if answer == SPDK_DOWN: + return "SPDK not running" + silent = now - t.first_fail + if silent >= constants.MULTI_ATTACH_LOST_AFTER_SEC: + return f"no answer for {silent:.1f}s" + return None + + def _observer_ok(self, cluster, nid, now) -> str: + """Why the takeover of ``nid`` must wait because this process may be + the one cut off; empty when it may go ahead.""" + cid = cluster.get_id() + with self.lock: + peers = {n: t for n, t in self.tracks.items() + if self.cluster_of.get(n) == cid} + failing = [n for n, t in peers.items() if t.first_fail is not None] + ftt = max(1, int(getattr(cluster, "max_fault_tolerance", 1) or 1)) + if len(failing) > ftt: + return (f"{len(failing)} nodes of the cluster fail their probes, more than it " + f"tolerates ({ftt}): this monitor may be the one cut off") + recent = constants.MULTI_ATTACH_LOST_AFTER_SEC + 2 * constants.MULTI_ATTACH_PROBE_TIMEOUT_SEC + if not any(n != nid and t.first_fail is None and now - t.last_ok <= recent + for n, t in peers.items()): + return "no other node of the cluster answered recently" + return "" + + def fire(self, node_id, reason, now) -> list: + """Start the takeover of every LUN ``node_id`` serves; the device ids + started. At most every REFIRE_SEC per node.""" + with self.lock: + t = self.tracks.get(node_id) + if t is None or (t.fired_at is not None and now - t.fired_at < REFIRE_SEC): + return [] + db = self._db() + node = db.get_storage_node_by_id(node_id) + if node.status not in WATCH_STATUSES: + return [] + cluster = db.get_cluster_by_id(node.cluster_id) + if not multi_attach.is_multi_attach(cluster): + return [] + why_not = self._observer_ok(cluster, node_id, now) + if why_not: + logger.warning("LUN watchdog: %s is lost (%s) but no takeover: %s", + node_id, reason, why_not) + return [] + devs = [dev for _o, dev in multi_attach.devices_served_by( + db.get_storage_nodes_by_cluster_id(node.cluster_id), node_id) + if dev.status in SERVING_STATES] + with self.lock: + t.fired_at = now + if not devs: + return [] + logger.warning("LUN watchdog: node %s lost (%s); taking over %d LUN(s): %s", + node_id, reason, len(devs), [d.get_id() for d in devs]) + started = [] + for dev in devs: + try: + self.start_takeover(dev.get_id(), node_id, f"node lost: {reason}") + started.append(dev.get_id()) + except Exception as e: # noqa: BLE001 + logger.error("LUN watchdog: takeover of %s not started: %s", dev.get_id(), e) + return started + + def forget(self, node_id): + with self.lock: + self.tracks.pop(node_id, None) + self.cluster_of.pop(node_id, None) + + # ------------------------------------------------------------ threads + + def _watch_node(self, node_id, nodes): + """Probe one node until it is no longer watched.""" + failing = False + while True: + node = nodes.get(node_id) + if node is None: + self.forget(node_id) + return + started = time.monotonic() + try: + answer = probe(node, failing) + now = time.monotonic() + failing = answer != OK + reason = self.record(node, answer, now) + if reason: + self.fire(node_id, reason, now) + except Exception as e: # noqa: BLE001 + logger.error("LUN watchdog: probe of %s failed: %s", node_id, e) + wait = started + constants.MULTI_ATTACH_PROBE_INTERVAL_SEC - time.monotonic() + if wait > 0: + time.sleep(wait) + + def run(self): + nodes: dict = {} + threads: dict[str, threading.Thread] = {} + while True: + try: + fresh = self.watched() + except Exception as e: # noqa: BLE001 + logger.error("LUN watchdog: watched nodes not read: %s", e) + fresh = None + if fresh is not None: + for nid in list(nodes): + if nid not in fresh: + nodes.pop(nid, None) + nodes.update(fresh) + for nid in fresh: + t = threads.get(nid) + if t is None or not t.is_alive(): + t = threading.Thread(target=self._watch_node, args=(nid, nodes), + name=f"lun-watch-{nid[:8]}", daemon=True) + threads[nid] = t + t.start() + for nid in [n for n, t in threads.items() if not t.is_alive()]: + threads.pop(nid, None) + time.sleep(REFRESH_SEC) + + +_started: Watchdog | None = None + + +def start(start_takeover) -> Watchdog: + """Start the watchdog thread of this process (once).""" + global _started + if _started is None: + _started = Watchdog(start_takeover) + threading.Thread(target=_started.run, name="lun-watchdog", daemon=True).start() + logger.info("LUN watchdog started (probe %.1fs, lost after %.1fs)", + constants.MULTI_ATTACH_PROBE_INTERVAL_SEC, + constants.MULTI_ATTACH_LOST_AFTER_SEC) + return _started diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index ce6f1f60f6..f3fa7f9518 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -2037,21 +2037,13 @@ def set_node_offline(node): logger.error("add_node_to_auto_restart for %s failed: %s", node.get_id(), ar_e) -# node id -> monotonic time it was first seen UNREACHABLE in the current -# episode (multi-attach LUN takeover trigger; see -# constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC). Cleared on any other status. -_unreachable_since: dict = {} - - def _take_over_luns_of_unreachable(node): """Multi-attach clusters (only): start the takeover of the LUNs a node - serves once it has been UNREACHABLE for MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC. - The takeover fences the node at the array first; each LUN gets one - thread (_start_lun_takeover is idempotent across monitor cycles).""" + serves as soon as it is UNREACHABLE. Usually the fast probe + (services/lun_watchdog) has started it seconds earlier; this is its + backstop. The takeover fences the node at the array first; each LUN gets + one thread (_start_lun_takeover is idempotent across monitor cycles).""" node_id = node.get_id() - since = _unreachable_since.setdefault(node_id, time.monotonic()) - if time.monotonic() - since < constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC: - return try: cluster = db.get_cluster_by_id(node.cluster_id) except Exception as e: # noqa: BLE001 @@ -2908,10 +2900,6 @@ def _spdk_is_dead(snode): def check_node(snode): snode = db.get_storage_node_by_id(snode.get_id()) - if snode.status != StorageNode.STATUS_UNREACHABLE: - # Ends an UNREACHABLE episode: a later one starts the multi-attach - # LUN takeover timer afresh (_take_over_luns_of_unreachable). - _unreachable_since.pop(snode.get_id(), None) if snode.status == StorageNode.STATUS_ONLINE: storage_network.publish_ports(snode, at_most_every=PORT_PUBLISH_INTERVAL_SEC) _reintegrate_fenced_luns(snode) @@ -3374,6 +3362,9 @@ def main(): logger.info("Starting node monitor") threads_maps: dict[str, threading.Thread] = {} _start_housekeeping() + # Multi-attach clusters only; idle otherwise. + from simplyblock_core.services import lun_watchdog + lun_watchdog.start(_start_lun_takeover) while True: clusters = db.get_clusters() diff --git a/tests/unit/test_array_lifecycle.py b/tests/unit/test_array_lifecycle.py index de835e6983..08a03fa341 100644 --- a/tests/unit/test_array_lifecycle.py +++ b/tests/unit/test_array_lifecycle.py @@ -804,6 +804,19 @@ def test_fence_is_forced_detach_and_waits_until_gone(self): self.assertTrue(self.p.fence("vol-0abc", "i-1")) self.stub.assert_no_pending_responses() + def test_fence_polls_fast(self): + slept = [] + self.p._sleep = slept.append + self._describe(self._vol(atts=[("i-1", "attached")])) + self.stub.add_response("detach_volume", {}, {"VolumeId": "vol-0abc", "InstanceId": "i-1", + "Force": True}) + self._describe(self._vol(atts=[("i-1", "detaching")])) + self._describe(self._vol(atts=[])) + self.assertTrue(self.p.fence("vol-0abc", "i-1")) + self.assertEqual(slept, [0.2]) + self.p.options["fence_poll_interval"] = 0.05 + self.assertEqual(self.p._fence_poll, 0.05) + def test_detach_not_attached_is_a_noop(self): self._describe(self._vol(atts=[("i-2", "attached")])) self.p.detach("vol-0abc", "i-1") diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index 315a5c7600..c673383051 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -231,6 +231,8 @@ def setUp(self): FakeArrayPlugin.reset() array.register_plugin(PLUGIN, FakeArrayPlugin) self.addCleanup(array.unregister_plugin, PLUGIN) + array.forget_plugin() + self.addCleanup(array.forget_plugin) self.events = [] for target, attr, kw in ( (lun_takeover, "DBController", {}), @@ -313,9 +315,10 @@ def test_fence_failure_means_no_takeover(self): db, _ = _bed() self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) self.assertEqual(self.dev(db).serving_node(), "node-a") + # The standby may have been probed (read-only, during the + # fence); nothing was built on any node. for n in db.nodes.values(): self.assertEqual(n.spdk.calls, []) - n.agent.lun_info.assert_not_called() def test_no_plugin_refuses(self): db, _ = _bed(plugin="") @@ -381,6 +384,56 @@ def test_rerun_after_partial_export_reuses_it(self): self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) self.assertEqual(self.dev(db).serving_node(), "node-b") + def test_standby_is_chosen_while_the_fence_runs(self): + import threading + db, _ = _bed() + probed = threading.Event() + db.nodes["node-b"].agent.lun_info.side_effect = ( + lambda *_a, **_k: (probed.set(), ([_lun_entry()], None))[1]) + seen = [] + + class SlowFence(FakeArrayPlugin): + def fence(self, lun_id, host_id): + seen.append(probed.wait(5)) + return super().fence(lun_id, host_id) + + array.register_plugin(PLUGIN, SlowFence) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(seen, [True]) + + def test_nothing_is_built_before_the_fence_is_acknowledged(self): + db, _ = _bed() + order = [] + b = db.nodes["node-b"] + create = b.spdk.bdev_aio_create + b.spdk.bdev_aio_create = lambda *a, **k: (order.append("build"), create(*a, **k))[1] + + class Fence(FakeArrayPlugin): + def fence(self, lun_id, host_id): + order.append("fence") + return super().fence(lun_id, host_id) + + array.register_plugin(PLUGIN, Fence) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(order, ["fence", "build"]) + + def test_array_connection_is_reused(self): + db, _ = _bed() + self.run_takeover(db) + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + self.assertEqual(self.run_takeover(db, failed="node-b"), lun_takeover.TAKEN_OVER) + self.assertEqual(len(FakeArrayPlugin.connected), 1) + + def test_fence_failure_retried_on_a_new_connection(self): + db, _ = _bed() + stale, fresh = MagicMock(), MagicMock() + stale.fence.side_effect = RuntimeError("ExpiredToken") + fresh.fence.return_value = True + with patch.object(array, "cached_plugin_for_cluster", side_effect=[stale, fresh]), patch.object(array, "forget_plugin") as forget: + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + forget.assert_called_once_with("cluster-1") + fresh.fence.assert_called_once_with(WWN, "host-node-a") + def test_concurrent_takeover_is_refused(self): db, _ = _bed() db.lock_holder = "another-runner" @@ -707,15 +760,14 @@ def test_takeover_cause_needs_an_online_serving_node(self): # --------------------------------------------- multi-attach bed findings, F1 class TestUnreachableTrigger(unittest.TestCase): - """The takeover starts once a node has been UNREACHABLE for - MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC, not when it is declared OFFLINE: - on the bed the OFFLINE verdict took ~6 min, far past the peers' hold.""" + """The regular monitor starts the takeover as soon as it sees a node + UNREACHABLE (backstop of services/lun_watchdog), not when it is declared + OFFLINE: on the bed the OFFLINE verdict took ~6 min, far past the peers' + hold. No confirmation wait: the fence makes an early takeover safe.""" def setUp(self): from simplyblock_core.services import storage_node_monitor as mon self.mon = mon - mon._unreachable_since.clear() - self.addCleanup(mon._unreachable_since.clear) self.db, self.dev = _bed() self.db.get_storage_node_by_id("node-a").status = StorageNode.STATUS_UNREACHABLE for target, attr, kw in ((mon, "db", {"new": self.db}), @@ -726,31 +778,24 @@ def setUp(self): self.start = started self.addCleanup(p.stop) - def _trigger(self, after): - node = self.db.get_storage_node_by_id("node-a") - with patch.object(self.mon.time, "monotonic", return_value=1000.0): - self.mon._take_over_luns_of_unreachable(node) - with patch.object(self.mon.time, "monotonic", return_value=1000.0 + after): - self.mon._take_over_luns_of_unreachable(node) - - def test_not_before_the_confirmation_time(self): - self._trigger(constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC - 1) - self.start.assert_not_called() - - def test_after_the_confirmation_time(self): - self._trigger(constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC) + def test_at_once(self): + self.mon._take_over_luns_of_unreachable(self.db.get_storage_node_by_id("node-a")) self.start.assert_called_once_with(DEV_ID, "node-a", "node unreachable") def test_regular_cluster_never(self): self.db.cluster.multi_attach = False - self._trigger(10 * constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC) + self.mon._take_over_luns_of_unreachable(self.db.get_storage_node_by_id("node-a")) self.start.assert_not_called() - def test_hold_covers_detection_trigger_and_takeover(self): - # ~20 s to see the node unreachable, the trigger delay, the takeover deadline + def test_no_confirmation_wait_left(self): + self.assertFalse(hasattr(constants, "MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC")) + self.assertFalse(hasattr(self.mon, "_unreachable_since")) + + def test_hold_covers_detection_and_takeover(self): + # ~20 s for the regular monitor to see the node unreachable, then the + # takeover deadline. self.assertGreater(constants.MULTI_ATTACH_HOLD_SEC, - 20 + constants.MULTI_ATTACH_UNREACHABLE_TAKEOVER_SEC - + constants.MULTI_ATTACH_TAKEOVER_DEADLINE_SEC) + 20 + constants.MULTI_ATTACH_TAKEOVER_DEADLINE_SEC) class TestRetireStaleLocalStack(unittest.TestCase): diff --git a/tests/unit/test_lun_watchdog.py b/tests/unit/test_lun_watchdog.py new file mode 100644 index 0000000000..d4e3c74dbb --- /dev/null +++ b/tests/unit/test_lun_watchdog.py @@ -0,0 +1,264 @@ +"""services/lun_watchdog: fast loss detection of nodes serving multi-attach +LUNs, and the immediate takeover of their LUNs. + + - probe: SPDK RPC first; the node agent decides between a killed SPDK, a + slow one and a node that is gone. + - decisions: SPDK down -> lost at once; no answer -> lost after + MULTI_ATTACH_LOST_AFTER_SEC; SPDK running -> never; an answer resets. + - fire: every LUN the node serves, rate-limited; never on a regular + cluster; never while this process may be the one cut off. + - watched: multi-attach clusters, nodes that serve a LUN. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import array, constants +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import lun_watchdog as wd +from tests.unit.test_lun_takeover import DEV_ID, _bed + +LOST = constants.MULTI_ATTACH_LOST_AFTER_SEC + + +class _Db: + """FakeDB of test_lun_takeover with get_clusters.""" + + def __init__(self, db): + self.inner = db + + def __getattr__(self, name): + return getattr(self.inner, name) + + def get_clusters(self): + return [self.inner.cluster] + + +def _setup(serving_a=True): + """Bed of test_lun_takeover with node-a ONLINE and serving its LUN.""" + db, dev = _bed() + a = db.nodes["node-a"] + a.status = StorageNode.STATUS_ONLINE + db.cluster.max_fault_tolerance = 1 + if not serving_a: + dev.serving_node_id = "node-b" + return _Db(db), dev + + +class TestProbe(unittest.TestCase): + + def _node(self, rpc_ok=True, agent=(True, None), agent_raises=False): + node = MagicMock() + node.rpc_port, node.cluster_id = 5260, "cluster-1" + rpc = MagicMock() + if not rpc_ok: + rpc.get_version.side_effect = ConnectionError("refused") + node.rpc_client.return_value = rpc + api = MagicMock() + if agent_raises: + api.spdk_process_is_up.side_effect = RuntimeError("timeout") + else: + api.spdk_process_is_up.return_value = agent + node.client.return_value = api + return node + + def test_rpc_answers(self): + node = self._node() + self.assertEqual(wd.probe(node), wd.OK) + node.client.assert_not_called() + kw = node.rpc_client.call_args.kwargs + self.assertEqual((kw["timeout"], kw["retry"]), (constants.MULTI_ATTACH_PROBE_TIMEOUT_SEC, 0)) + + def test_spdk_killed(self): + self.assertEqual(wd.probe(self._node(False, (False, "socket down"))), wd.SPDK_DOWN) + + def test_spdk_slow(self): + self.assertEqual(wd.probe(self._node(False, (True, None))), wd.SPDK_UP) + + def test_node_gone(self): + node = self._node(False, agent_raises=True) + self.assertEqual(wd.probe(node), wd.NO_ANSWER) + kw = node.client.call_args.kwargs + self.assertEqual((kw["retry"], kw["connect_retry"]), (0, 0)) + + def test_agent_without_verdict(self): + self.assertEqual(wd.probe(self._node(False, (None, "error"))), wd.NO_ANSWER) + + def test_failing_asks_both_at_once(self): + node = self._node(False, agent_raises=True) + self.assertEqual(wd.probe(node, failing=True), wd.NO_ANSWER) + node.client.assert_called_once() + self.assertEqual(wd.probe(self._node(True), failing=True), wd.OK) + + +class TestDecisions(unittest.TestCase): + + def setUp(self): + self.db, self.dev = _setup() + self.w = wd.Watchdog(MagicMock(), db=self.db) + self.a = self.db.nodes["node-a"] + + def test_spdk_down_is_lost_at_once(self): + self.assertEqual(self.w.record(self.a, wd.SPDK_DOWN, 100.0), "SPDK not running") + + def test_no_answer_is_lost_after_the_window(self): + # Probes of a node that does not answer take PROBE_TIMEOUT each. + t = 100.0 + step = constants.MULTI_ATTACH_PROBE_TIMEOUT_SEC + reasons = [] + while t < 100.0 + LOST + step: + reasons.append(self.w.record(self.a, wd.NO_ANSWER, t)) + t += step + self.assertTrue(all(r is None for r in reasons[:-1]), reasons) + self.assertTrue(reasons[-1].startswith("no answer for")) + # Detection inside the 9 s stall budget with room for the takeover. + self.assertLessEqual(LOST + 2 * step, 4.0) + + def test_running_spdk_is_never_lost(self): + for t in range(60): + self.assertIsNone(self.w.record(self.a, wd.SPDK_UP, 100.0 + t)) + + def test_an_answer_within_the_window_cancels(self): + self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 100.0)) + self.assertIsNone(self.w.record(self.a, wd.OK, 100.0 + LOST - 0.5)) + self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 100.0 + LOST + 0.1)) + + +class TestFire(unittest.TestCase): + + def setUp(self): + self.db, self.dev = _setup() + self.start = MagicMock() + self.w = wd.Watchdog(self.start, db=self.db) + # Every node probed fine at t=100. + for n in self.db.nodes.values(): + self.w.record(n, wd.OK, 100.0) + + def _lose(self, nid, t=101.0): + reason = self.w.record(self.db.nodes[nid], wd.SPDK_DOWN, t) + return self.w.fire(nid, reason, t) + + def test_takes_over_every_lun_the_node_serves(self): + self.assertEqual(self._lose("node-a"), [DEV_ID]) + self.start.assert_called_once_with(DEV_ID, "node-a", "node lost: SPDK not running") + + def test_rate_limited(self): + self._lose("node-a", 101.0) + self._lose("node-a", 102.0) + self.assertEqual(self.start.call_count, 1) + for nid in ("node-b", "node-c", "node-d"): + self.w.record(self.db.nodes[nid], wd.OK, 101.0 + wd.REFIRE_SEC) + self._lose("node-a", 101.0 + wd.REFIRE_SEC) + self.assertEqual(self.start.call_count, 2) + + def test_lun_served_elsewhere_is_not_touched(self): + self.dev.serving_node_id = "node-b" + self.assertEqual(self._lose("node-a"), []) + self.start.assert_not_called() + + def test_failed_device_is_not_taken_over(self): + self.dev.status = NVMeDevice.STATUS_UNAVAILABLE + self.assertEqual(self._lose("node-a"), []) + + def test_regular_cluster_never(self): + self.db.cluster.multi_attach = False + self.assertEqual(self._lose("node-a"), []) + + def test_node_in_restart_is_left_to_the_restart(self): + self.db.nodes["node-a"].status = StorageNode.STATUS_RESTARTING + self.assertEqual(self._lose("node-a"), []) + + def test_not_when_more_nodes_fail_than_tolerated(self): + # This monitor lost its own network: every node looks gone. + self.w.record(self.db.nodes["node-b"], wd.NO_ANSWER, 100.5) + self.assertEqual(self._lose("node-a"), []) + self.start.assert_not_called() + + def test_two_lost_nodes_with_ftt2(self): + self.db.cluster.max_fault_tolerance = 2 + self.w.record(self.db.nodes["node-b"], wd.NO_ANSWER, 100.5) + self.assertEqual(self._lose("node-a"), [DEV_ID]) + + def test_not_when_no_other_node_answered_recently(self): + self.assertEqual(self._lose("node-a", 100.0 + LOST + 10), []) + self.start.assert_not_called() + + +class TestWatched(unittest.TestCase): + + def test_nodes_serving_a_lun(self): + db, _ = _setup() + w = wd.Watchdog(MagicMock(), db=db) + with patch.object(array, "cached_plugin_for_cluster"): + self.assertEqual(sorted(w.watched()), ["node-a"]) + + def test_takeover_moves_the_watch(self): + db, _ = _setup(serving_a=False) + w = wd.Watchdog(MagicMock(), db=db) + with patch.object(array, "cached_plugin_for_cluster"): + self.assertEqual(sorted(w.watched()), ["node-b"]) + + def test_regular_and_activating_clusters_are_not_watched(self): + for change in ("regular", "activation"): + with self.subTest(change=change): + db, _ = _setup() + if change == "regular": + db.cluster.multi_attach = False + else: + db.cluster.status = Cluster.STATUS_IN_ACTIVATION + w = wd.Watchdog(MagicMock(), db=db) + self.assertEqual(w.watched(), {}) + + def test_offline_node_is_not_watched(self): + db, _ = _setup() + db.nodes["node-a"].status = StorageNode.STATUS_OFFLINE + w = wd.Watchdog(MagicMock(), db=db) + with patch.object(array, "cached_plugin_for_cluster"): + self.assertEqual(w.watched(), {}) + + def test_array_connection_is_warmed(self): + db, _ = _setup() + db.cluster.array_plugin = "fake-test" + w = wd.Watchdog(MagicMock(), db=db) + with patch.object(array, "cached_plugin_for_cluster") as conn, \ + patch.object(wd.threading, "Thread") as thread: + thread.side_effect = lambda target, **_k: MagicMock(start=target) + w.watched() + w.watched() + conn.assert_called_once() + self.assertEqual(conn.call_args.kwargs["max_age"], constants.ARRAY_PLUGIN_CACHE_SEC / 2) + + +class TestPluginCache(unittest.TestCase): + + def setUp(self): + from simplyblock_core.array.fake import FakeArrayPlugin + self.cls = FakeArrayPlugin + FakeArrayPlugin.reset() + array.register_plugin("fake-cache", FakeArrayPlugin) + self.addCleanup(array.unregister_plugin, "fake-cache") + array.forget_plugin() + self.addCleanup(array.forget_plugin) + self.cluster = Cluster() + self.cluster.uuid = "cluster-c" + self.cluster.array_plugin = "fake-cache" + + def test_reused_until_the_configuration_changes(self): + p1 = array.cached_plugin_for_cluster(self.cluster) + self.assertIs(array.cached_plugin_for_cluster(self.cluster), p1) + self.cluster.array_options = {"x": 1} + self.assertIsNot(array.cached_plugin_for_cluster(self.cluster), p1) + self.assertEqual(len(self.cls.connected), 2) + + def test_expiry_and_forget(self): + p1 = array.cached_plugin_for_cluster(self.cluster) + self.assertIsNot(array.cached_plugin_for_cluster(self.cluster, max_age=0), p1) + p2 = array.cached_plugin_for_cluster(self.cluster) + array.forget_plugin("cluster-c") + self.assertIsNot(array.cached_plugin_for_cluster(self.cluster), p2) + + +if __name__ == "__main__": + unittest.main() From 8d40d8a5a764edf0023fc96293ded4f0313f5fb3 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 10:54:03 +0100 Subject: [PATCH 23/50] fix(multi-attach): count a node's silence from its first unanswered probe The watchdog started the 2 s window when the first failed probe returned, after its RPC and agent timeouts (1.6 s): on the FTT2 bed a network cut was declared lost 4.1 s after it began instead of ~2.5 s. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/services/lun_watchdog.py | 12 +++++++----- tests/unit/test_lun_watchdog.py | 7 +++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/simplyblock_core/services/lun_watchdog.py b/simplyblock_core/services/lun_watchdog.py index 3ba5d97449..9cbac1423c 100644 --- a/simplyblock_core/services/lun_watchdog.py +++ b/simplyblock_core/services/lun_watchdog.py @@ -174,9 +174,11 @@ def _connect(): # ---------------------------------------------------------- decisions - def record(self, node, answer, now) -> str | None: - """Note a probe answer of ``node`` at ``now``; the reason the node is - lost, or None.""" + def record(self, node, answer, now, sent=None) -> str | None: + """Note a probe answer of ``node`` at ``now`` to a probe ``sent`` then + (default ``now``); the reason the node is lost, or None. The silence + counts from when the first unanswered probe was sent: a probe that + times out returns only after its timeouts.""" nid = node.get_id() with self.lock: self.cluster_of[nid] = node.cluster_id @@ -189,7 +191,7 @@ def record(self, node, answer, now) -> str | None: t.first_fail, t.fired_at, t.last_ok = None, None, now return None if t.first_fail is None: - t.first_fail = now + t.first_fail = now if sent is None else sent if answer == SPDK_UP: return None if answer == SPDK_DOWN: @@ -274,7 +276,7 @@ def _watch_node(self, node_id, nodes): answer = probe(node, failing) now = time.monotonic() failing = answer != OK - reason = self.record(node, answer, now) + reason = self.record(node, answer, now, started) if reason: self.fire(node_id, reason, now) except Exception as e: # noqa: BLE001 diff --git a/tests/unit/test_lun_watchdog.py b/tests/unit/test_lun_watchdog.py index d4e3c74dbb..17748a5872 100644 --- a/tests/unit/test_lun_watchdog.py +++ b/tests/unit/test_lun_watchdog.py @@ -116,6 +116,13 @@ def test_no_answer_is_lost_after_the_window(self): # Detection inside the 9 s stall budget with room for the takeover. self.assertLessEqual(LOST + 2 * step, 4.0) + def test_silence_counts_from_the_first_unanswered_probe(self): + # The first probe was sent at 100 and gave up (RPC + agent timeouts) + # at 101.6; the next one, asking both at once, ends at 102.4. + self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 101.6, sent=100.0)) + self.assertTrue(self.w.record(self.a, wd.NO_ANSWER, 102.4, sent=101.6) + .startswith("no answer for 2.4s")) + def test_running_spdk_is_never_lost(self): for t in range(60): self.assertIsNone(self.w.record(self.a, wd.SPDK_UP, 100.0 + t)) From 9277841f1a289688bb617a261674502196eab6fa Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 12:02:44 +0100 Subject: [PATCH 24/50] fix(multi-attach): restart nodes up to the cluster's FTT, not its npcs The auto-restart guard refused a node while more other nodes were down than distr_npcs. A multi-attach cluster runs 1+0 (npcs 0) and tolerates max_fault_tolerance node losses through LUN takeovers: on the FTT2 bed a network cut plus an SPDK kill left both nodes offline for good, each blocking the other's restart while the cluster stayed DEGRADED. Regular clusters keep the npcs rule. Co-Authored-By: Claude Opus 5.5 --- .../controllers/tasks_controller.py | 16 ++++- tests/unit/test_multi_attach_auto_restart.py | 69 +++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_multi_attach_auto_restart.py diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index e9c7dad063..5936dcecce 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -8,7 +8,13 @@ from collections.abc import Callable from typing import NamedTuple -from simplyblock_core import constants, db_controller, storage_node_ops, utils +from simplyblock_core import ( + constants, + db_controller, + multi_attach, + storage_node_ops, + utils, +) from simplyblock_core.controllers import ( activation_gate, device_controller, @@ -848,7 +854,13 @@ def add_node_to_auto_restart(node): for sn in db.get_storage_nodes_by_cluster_id(node.cluster_id): if node.get_id() != sn.get_id() and sn.status != StorageNode.STATUS_ONLINE and node.mgmt_ip != sn.mgmt_ip: offline_nodes += 1 - if offline_nodes > cluster.distr_npcs and cluster.status != Cluster.STATUS_SUSPENDED: + # A multi-attach cluster runs 1+0 (npcs 0) and tolerates FTT node + # losses through LUN takeovers: with npcs, two nodes down at FTT 2 + # blocked each other's restart for good while the cluster stayed + # DEGRADED (2026-11-20 FTT2 bed, network cut + SPDK kill). + tolerated = (cluster.max_fault_tolerance if multi_attach.is_multi_attach(cluster) + else cluster.distr_npcs) + if offline_nodes > tolerated and cluster.status != Cluster.STATUS_SUSPENDED: logger.info("Node found that is not online, skip node auto restart") return False return _add_task(JobSchedule.FN_NODE_RESTART, node.cluster_id, node.get_id(), "", max_retry=11) diff --git a/tests/unit/test_multi_attach_auto_restart.py b/tests/unit/test_multi_attach_auto_restart.py new file mode 100644 index 0000000000..e9e14a9197 --- /dev/null +++ b/tests/unit/test_multi_attach_auto_restart.py @@ -0,0 +1,69 @@ +"""Auto-restart admission on multi-attach clusters: they run 1+0 (npcs 0) and +tolerate ``max_fault_tolerance`` node losses, so two nodes down at FTT 2 must +each be restarted (2026-11-20 FTT2 bed: network cut + SPDK kill left both +offline for good, each blocking the other). Regular clusters keep the npcs +rule.""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core.controllers import tasks_controller as tc +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode + + +def _node(nid, status): + n = StorageNode() + n.uuid = nid + n.cluster_id = "c1" + n.status = status + n.mgmt_ip = f"10.0.0.{len(nid)}{nid[-1]}" + return n + + +def _cluster(multi, ftt, npcs): + c = Cluster() + c.uuid = "c1" + c.status = Cluster.STATUS_DEGRADED + c.multi_attach = multi + c.max_fault_tolerance = ftt + c.distr_ndcs, c.distr_npcs = (1, npcs) + return c + + +class TestAutoRestartTolerance(unittest.TestCase): + + def _queue(self, cluster, offline_others): + nodes = [_node("n0", StorageNode.STATUS_OFFLINE)] + nodes += [_node(f"n{i + 1}", StorageNode.STATUS_OFFLINE) for i in range(offline_others)] + nodes += [_node(f"m{i}", StorageNode.STATUS_ONLINE) for i in range(3)] + db = MagicMock() + db.get_storage_node_by_id.return_value = nodes[0] + db.get_cluster_by_id.return_value = cluster + db.get_storage_nodes_by_cluster_id.return_value = nodes + with patch.object(tc, "db", db), \ + patch.object(tc, "is_auto_restart_paused", return_value=False), \ + patch.object(tc, "_add_task", return_value="task") as add: + tc.add_node_to_auto_restart(nodes[0]) + return add.called + + def test_multi_attach_ftt2_restarts_with_another_node_down(self): + self.assertTrue(self._queue(_cluster(True, 2, 0), offline_others=1)) + + def test_multi_attach_ftt2_holds_beyond_its_tolerance(self): + # Same rule as npcs on a regular cluster: held once more OTHER nodes + # are down than the cluster tolerates. + self.assertTrue(self._queue(_cluster(True, 2, 0), offline_others=2)) + self.assertFalse(self._queue(_cluster(True, 2, 0), offline_others=3)) + + def test_multi_attach_ftt1(self): + self.assertTrue(self._queue(_cluster(True, 1, 0), offline_others=1)) + self.assertFalse(self._queue(_cluster(True, 1, 0), offline_others=2)) + + def test_regular_cluster_keeps_the_npcs_rule(self): + self.assertTrue(self._queue(_cluster(False, 1, 1), offline_others=1)) + self.assertFalse(self._queue(_cluster(False, 2, 1), offline_others=2)) + + +if __name__ == "__main__": + unittest.main() From e8f8d76d3a74520e407ef84054d3fcbebe3c2b1b Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 13:34:07 +0100 Subject: [PATCH 25/50] fix(multi-attach): never resolve an absent array LUN by its old kernel name resolve_lblk_entries falls back to the stored kernel name when a device's serial is not on the host. On a multi-attach cluster an absent LUN is one the node is fenced from or detached, and its old name can be another disk: on the FTT1 bed a node hard-reset while fenced from its LUNs came back with its journal volume at the LUN's old name nvme2n1; restart took the journal up as a data device, bdev_examine failed with EEXIST on every attempt and the node never returned (without that error it would have been treated as a new data device). Multi-attach clusters (restart and add-node) now match entries with a serial by serial/partuuid only; regular clusters keep the name fallback. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/storage_node_ops.py | 6 ++++-- simplyblock_core/utils/__init__.py | 13 ++++++++++--- tests/unit/test_lblk_eligibility.py | 24 ++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index edbc05c13d..9cf6654bd6 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4718,7 +4718,8 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, logger.error(f"Failed to list block devices on {node_addr}: {err}") return False lblk_resolved, missing = utils.resolve_lblk_entries( - lblk_configured, blk_inventory) + lblk_configured, blk_inventory, + name_fallback=not multi_attach.is_multi_attach(cluster)) if missing: logger.error( f"Configured block device(s) not found on host: " @@ -9968,7 +9969,8 @@ def _restart_storage_node_impl( logger.error(f"Failed to list block devices: {blk_err}") return False lblk_resolved, lblk_missing = utils.resolve_lblk_entries( - snode.lblk_devices, blk_inventory) + snode.lblk_devices, blk_inventory, + name_fallback=not multi_attach.is_multi_attach(cluster)) for entry in lblk_missing: logger.warning(f"Configured block device {entry.get('path') or entry.get('name')} " f"(serial {entry.get('serial')}) not found on host") diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index c61a082508..435fe47829 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1656,7 +1656,7 @@ def _unsafe_live_device_reason(dev): return None -def resolve_lblk_entries(configured_entries, host_devices): +def resolve_lblk_entries(configured_entries, host_devices, *, name_fallback=True): """Match the node's configured lblk devices against the live host inventory, SERIAL-FIRST: kernel names shift across reboots, so the stored name is only a fallback for devices without a resolvable serial. Partition @@ -1677,7 +1677,14 @@ def resolve_lblk_entries(configured_entries, host_devices): (mounted, held, or backing the root filesystem) is demoted to `missing` rather than returned as resolved — a fallback match by name/partuuid must clear the same safety bar `filter_eligible_block_devices` enforces - at add-node time, not just "something is attached at that name now".""" + at add-node time, not just "something is attached at that name now". + + ``name_fallback=False`` (multi-attach clusters): an entry with a serial is + matched by serial or partuuid only. An array LUN keeps its identity; when + it is absent the node is fenced from it or it is detached, and its old + kernel name may now be another disk of the host: on the FTT1 bed, after a + hard reset of a node fenced from its LUN, the LUN's name nvme2n1 was the + node's journal volume, which the restart took up as a data device.""" by_path = _index_by_selector(host_devices) by_serial = {d["serial"]: d for d in host_devices} by_name = {d["name"]: d for d in host_devices} @@ -1699,7 +1706,7 @@ def resolve_lblk_entries(configured_entries, host_devices): if live is None and entry.get("partuuid"): live = by_partuuid.get(entry["partuuid"].lower()) matched_by = "partuuid" - if live is None: + if live is None and (name_fallback or not entry.get("serial")): live = by_name.get(entry.get("name")) matched_by = "name" if live is None: diff --git a/tests/unit/test_lblk_eligibility.py b/tests/unit/test_lblk_eligibility.py index 1f93552173..ee04d55f92 100644 --- a/tests/unit/test_lblk_eligibility.py +++ b/tests/unit/test_lblk_eligibility.py @@ -272,6 +272,30 @@ def test_name_fallback_when_serial_unknown(self): names = {e["name"] for e in resolved} self.assertEqual(names, {"sdb", "sdc"}) + def test_no_name_fallback_for_array_luns(self): + # Multi-attach: S1 is a LUN the node is fenced from; after a reboot + # its old name sdb is another disk (FTT1 bed: the journal volume). + host = [_blk("sdb", serial="JOURNAL"), _blk("sdc", serial="S2")] + resolved, missing = utils.resolve_lblk_entries(self.CONFIGURED, host, + name_fallback=False) + self.assertEqual([e["serial"] for e in resolved], ["S2"]) + self.assertEqual([e["serial"] for e in missing], ["S1"]) + + def test_serial_less_entry_still_falls_back_by_name(self): + configured = [{"name": "sdb", "serial": "", "by_id": "", "size": 100, "numa": 0}] + resolved, missing = utils.resolve_lblk_entries(configured, [_blk("sdb", serial="X")], + name_fallback=False) + self.assertEqual((len(resolved), missing), (1, [])) + + def test_node_paths_turn_name_fallback_off_on_multi_attach(self): + from simplyblock_core import storage_node_ops + from tests._source import code_of + for fn in (storage_node_ops.add_node, storage_node_ops._restart_storage_node_impl): + code = code_of(fn) + self.assertIn("resolve_lblk_entries(", code, fn.__name__) + self.assertIn("name_fallback=not multi_attach.is_multi_attach(cluster)", code, + fn.__name__) + def test_missing_device_reported(self): host = [_blk("sdc", serial="S2")] resolved, missing = utils.resolve_lblk_entries(self.CONFIGURED, host) From 22cff864cfab9fc372674c115f3db69086abbe6e Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 14:03:05 +0100 Subject: [PATCH 26/50] feat(multi-attach): fence by persistent reservation; 3 s client keep-alive Takeover fence by reservation (ultra #236 LUN failover): every multi-attach alceml registers a key (node cntlid block + 1 in the upper 32 bits), the serving instance holds the reservation (Write Exclusive - Registrants Only), and a takeover opens the standby's alceml active with its key, which preempts the failed node in milliseconds. The takeover waits until alceml_get_role reports it active, exports and switches paths; the array fence (EBS force-detach, 0.5-4 s, slower with several LUNs of one node) follows in the background and the node is recorded fenced. If the alceml does not take the reservation, the takeover fences at the array and opens it again. Whether a LUN uses reservations is decided once, when it is marked a LUN (NVMeDevice.lun_reservations: MULTI_ATTACH_RESERVATIONS, the array plugin's supports_reservations, no fallback-mode checksum), because a LUN formatted with a key cannot be opened without one. Client keep-alive 3 s on multi-attach clusters (regular clusters keep 8 s): a hard-reset or cut-off node's own volumes waited 10-14 s for the host's keep-alive. The CSI driver forwards the value only for two-node clusters so far. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/constants.py | 15 +++ .../controllers/device_controller.py | 2 +- simplyblock_core/controllers/lun_takeover.py | 74 +++++++++++- simplyblock_core/models/nvme_device.py | 6 + simplyblock_core/multi_attach.py | 56 ++++++++- simplyblock_core/rpc_client.py | 15 ++- simplyblock_core/storage_node_ops.py | 2 +- simplyblock_core/utils/nvme.py | 6 + tests/unit/test_lun_takeover.py | 107 ++++++++++++++++++ tests/unit/test_two_node_c1_fixes.py | 12 ++ 10 files changed, 284 insertions(+), 11 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index c1b0c121bd..0ebbe7ad52 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -1353,6 +1353,21 @@ def get_config_var(name, default=None): # peers' hold, or their I/O fails before the new path exists; it gives up at # this deadline (seconds from its start) and the device goes unavailable. MULTI_ATTACH_TAKEOVER_DEADLINE_SEC = 90 +# Fence a lost node's LUN with an NVMe/SCSI persistent reservation instead of +# waiting for the array to detach it: every multi-attach alceml registers a +# key (multi_attach.reservation_key), the owner holds the reservation, and a +# takeover's alceml preempts it at open (ultra #236), in milliseconds. The +# array fence (EBS force-detach: 0.5-4 s, longer with several LUNs of one +# node) then follows off the I/O path. Needs an array plugin that reports +# supports_reservations and an ultra data plane with LUN failover. +MULTI_ATTACH_RESERVATIONS = True +# A takeover waits this long for its alceml to report the reservation taken. +MULTI_ATTACH_ACTIVATION_WAIT_SEC = 5 +# Client keep-alive on multi-attach clusters: a node lost without closing its +# connections (hard reset, network cut) leaves its own volumes' paths dead +# until the host's keep-alive notices, 1-2 intervals: 8 s took 10-14 s on the +# bed. 3 s bounds that to 3-6 s. +MULTI_ATTACH_CLIENT_KEEP_ALIVE_TO = 3 # Fast loss detection of a node serving LUNs (services/lun_watchdog). With # 1+0 every volume waits on a lost node's LUN until its takeover, so the # takeover starts from this probe, not from the node's status: the OFFLINE diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index 4fa68e357e..20abaa60b5 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -804,7 +804,7 @@ def _def_create_device_stack(device_obj, snode, force=False, clear_data=False): checksum_method=checksum_method, cache_size=cache_size, cache_eviction_threshold=cache_eviction_threshold, - **multi_attach.alceml_create_kwargs(device_obj), + **multi_attach.alceml_create_kwargs(device_obj, snode, cluster), ) except RPCException as e: logger.error("Failed to create alceml bdev %s: %s", alceml_name, e) diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index 454b5341a8..93d8e4e8cd 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -227,10 +227,15 @@ def _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix=f"standby-{device_id[:8]}") choice = pool.submit(_choose_standby, owner, dev, failed_node_id, nodes_by_id, deadline) pool.shutdown(wait=False) + by_reservation = False try: - # 1. Fence -- before anything else is changed. - _fence(cluster, dev, failed) - _record_fenced(db, owner, device_id, failed_node_id) + # 1. Fence -- before anything else is changed. With reservations the + # standby's alceml fences at open (preempting the failed node's + # reservation, step 4) and the array fence follows off the I/O path. + by_reservation = _fence_by_reservation(cluster, dev) + if not by_reservation: + _fence(cluster, dev, failed) + _record_fenced(db, owner, device_id, failed_node_id) _check_deadline(deadline, "choosing a standby") # 2 + 3. A standby that can read the LUN. @@ -245,7 +250,24 @@ def _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) _check_deadline(deadline, f"exporting the LUN on {standby.get_id()}") # 4. Open-only export on the standby. - export = _export_on_standby(db, cluster, standby, dev, path, deadline) + try: + export = _export_on_standby(db, cluster, standby, dev, path, deadline) + except TakeoverError as e: + if not by_reservation: + raise + # The reservation was not taken: fence at the array, then once more. + logger.warning("LUN takeover of %s: fence by reservation failed (%s); fencing " + "at the array", device_id, e) + by_reservation = False + _fence(cluster, dev, failed) + _record_fenced(db, owner, device_id, failed_node_id) + _check_deadline(deadline, f"exporting the LUN on {standby.get_id()} again") + export = _export_on_standby(db, cluster, standby, dev, path, deadline) + if by_reservation: + # Fenced by the reservation: recorded now, detached at the array + # in the background (the attachment reconcile detaches it too). + _record_fenced(db, owner, device_id, failed_node_id) + _fence_in_background(cluster, dev, failed) _check_deadline(deadline, "recording the takeover") # 5 (design step 6). Record, atomically. @@ -319,6 +341,44 @@ def _fence(cluster, dev, failed): logger.info("LUN %s: node %s (%s) fenced at the array", lun, failed.get_id(), host) +def _fence_by_reservation(cluster, dev) -> bool: + """Whether this takeover fences by persistent reservation: the device's + alceml instances run with keys (NVMeDevice.lun_reservations, decided + with the array's capability when the LUN was added).""" + return multi_attach.reservations_enabled(cluster, dev) + + +def _fence_in_background(cluster, dev, failed): + """The array fence after a fence by reservation: off the I/O path, for + good measure (the reservation already rejects the node's writes).""" + def _run(): + try: + _fence(cluster, dev, failed) + except TakeoverError as e: + logger.warning("LUN %s: array fence of %s after the reservation failed: %s; the " + "attachment reconcile detaches it", dev.lun_wwn, failed.get_id(), e) + threading.Thread(target=_run, name=f"fence-{dev.get_id()[:8]}", daemon=True).start() + + +def _wait_active(rpc, alceml, deadline): + """Wait until ``alceml`` (created with a reservation key, not standby) + holds the reservation and serves. Raises TakeoverError.""" + limit = min(deadline, time.monotonic() + constants.MULTI_ATTACH_ACTIVATION_WAIT_SEC) + last = None + while True: + try: + last = rpc.alceml_get_role(alceml) or {} + except Exception as e: # noqa: BLE001 + last = {"error": str(e)} + if last.get("role") == "active" and last.get("state") == "active": + logger.info("LUN takeover: %s holds the reservation (epoch %s, activation %s ms)", + alceml, last.get("epoch"), last.get("last_activation_ms")) + return + if last.get("state") == "act_failed" or time.monotonic() >= limit: + raise TakeoverError(f"alceml {alceml} did not take the reservation: {last}") + time.sleep(0.05) + + def _record_fenced(db, owner, device_id, node_id): def _mut(n): d = _device_on_owner(n, device_id) @@ -424,6 +484,7 @@ def _export_on_standby(db, cluster, standby, dev, path, deadline) -> dict: raise TakeoverError(f"base bdev {base} on {sid} is {size} bytes, the device {dev.size}") alceml = dev.alceml_bdev or f"alceml_{dev.get_id()}" + kwargs = multi_attach.alceml_create_kwargs(dev, standby, cluster) if not rpc.bdev_get(alceml): checksum_method, cache_size, cache_eviction = utils.alceml_checksum_params(cluster, dev) ok = standby.create_alceml( @@ -436,11 +497,14 @@ def _export_on_standby(db, cluster, standby, dev, path, deadline) -> dict: cache_size=cache_size, cache_eviction_threshold=cache_eviction, force_4k_atomic=getattr(cluster, 'atomic_4k', False), - **multi_attach.alceml_create_kwargs(dev), + **kwargs, ) if not ok: raise TakeoverError(f"alceml {alceml} not opened on {sid}") created.append(("alceml", alceml)) + if kwargs.get("reservation_key"): + # Opened active with a key: it preempts the old holder (fence). + _wait_active(rpc, alceml, deadline) try: from simplyblock_core.controllers import qos_controller rpc.alceml_set_qos_weights(qos_controller.get_qos_weights_list(cluster.get_id())) diff --git a/simplyblock_core/models/nvme_device.py b/simplyblock_core/models/nvme_device.py index 951eb874b4..ec2f92ec15 100644 --- a/simplyblock_core/models/nvme_device.py +++ b/simplyblock_core/models/nvme_device.py @@ -156,6 +156,12 @@ class NVMeDevice(BaseModel): #: unfences on its own: a fenced node's host may still hold queued #: writes. Such a node is no takeover candidate for this LUN. fenced_node_ids: list = default_factory(list) + #: The LUN's alceml instances run with persistent reservation keys + #: (multi_attach.reservation_key): the serving one holds the reservation + #: and a takeover fences by preempting it. Decided once, when the device + #: is first marked a LUN (multi_attach.mark_lun_devices): a LUN formatted + #: with a key cannot be opened without one. + lun_reservations: bool = False #: The array's id of the LUN when the cluster's array plugin created it #: (controllers/array_controller); "" for a LUN the operator presented. #: Only such LUNs are attached, detached, grown and deleted by simplyblock. diff --git a/simplyblock_core/multi_attach.py b/simplyblock_core/multi_attach.py index b10b43f3ed..ac17b752d5 100644 --- a/simplyblock_core/multi_attach.py +++ b/simplyblock_core/multi_attach.py @@ -21,10 +21,13 @@ from __future__ import annotations +import logging import uuid as uuid_mod from simplyblock_core import constants +logger = logging.getLogger(__name__) + #: The NQN infix that marks a host-independent multi-attach device subsystem. LUN_NQN_INFIX = ":lun:" @@ -144,12 +147,58 @@ def attach_controller_kwargs(device) -> dict: } -def alceml_create_kwargs(device) -> dict: +def reservation_key(node) -> int: + """The persistent reservation key of ``node``'s alceml instances on array + LUNs: a node id unique in the cluster in the upper 32 bits (its + controller-ID block + 1, never 0); the instance sets the lower 32 bits + (its activation epoch). 0 when the node has no block yet.""" + block = getattr(node, "cntlid_block", -1) + if not isinstance(block, int) or block < 0: + return 0 + return (block + 1) << 32 + + +def reservations_supported(cluster, device) -> bool: + """Whether a new LUN device gets reservations (NVMeDevice.lun_reservations): + MULTI_ATTACH_RESERVATIONS on, the cluster's array honours reservations + (plugin capability supports_reservations), and the device's inline + checksum is not in fallback mode (the data plane refuses a key with + checksum_validation_method 2).""" + if not (constants.MULTI_ATTACH_RESERVATIONS and is_multi_attach(device)): + return False + from simplyblock_core import array, utils + if utils.alceml_checksum_params(cluster, device)[0] == 2: + return False + if not getattr(cluster, "array_plugin", ""): + return False + try: + caps = array.cached_plugin_for_cluster(cluster).capabilities() or {} + except Exception as e: # noqa: BLE001 + logger.warning("Array capabilities unavailable (%s); LUN %s without reservations", + e, getattr(device, "lun_wwn", "")) + return False + return bool(caps.get("supports_reservations")) + + +def reservations_enabled(cluster, device) -> bool: + """Whether the alceml of ``device`` runs with a reservation key: as + decided when it was marked a LUN (NVMeDevice.lun_reservations), the same + for every open of it.""" + return is_multi_attach(device) and getattr(device, "lun_reservations", False) is True + + +def alceml_create_kwargs(device, node=None, cluster=None) -> dict: """Extra ``bdev_alceml_create`` kwargs for a multi-attach device's alceml - (the stuck-I/O timeout); nothing otherwise.""" + on ``node``: the stuck-I/O timeout, and the reservation key when + reservations are enabled (reservations_enabled); nothing otherwise.""" if not is_multi_attach(device): return {} - return {"stuck_io_timeout_ms": constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS} + out = {"stuck_io_timeout_ms": constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS} + if node is not None and cluster is not None and reservations_enabled(cluster, device): + key = reservation_key(node) + if key: + out["reservation_key"] = key + return out def mark_lun_devices(cluster, snode, devices, blk_entries) -> None: @@ -167,6 +216,7 @@ def mark_lun_devices(cluster, snode, devices, blk_entries) -> None: if entry.get("journal"): continue dev.multi_attach = True + dev.lun_reservations = reservations_supported(cluster, dev) # A LUN the array plugin created carries the identity the plugin # reported for it (array_controller.provision_node_lun); it is one of # the WWN / serial the host reports, so lun_keys matches either way. diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 60d8803c70..6de3b0a8e2 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -980,7 +980,8 @@ def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3, alceml_cpu_mask="", alceml_worker_cpu_mask="", pba_page_size=2097152, write_protection=False, full_page_unmap=False, checksum_method=0, cache_size=0, cache_eviction_threshold=0, - force_4k_atomic=False, stuck_io_timeout_ms=0): + force_4k_atomic=False, stuck_io_timeout_ms=0, + reservation_key=0, standby=False): params = { "name": alceml_name, "cntr_path": nvme_name, @@ -1022,7 +1023,19 @@ def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3, # devices, whose array LUN needs longer than multipath failover. if stuck_io_timeout_ms: params["stuck_io_timeout_ms"] = int(stuck_io_timeout_ms) + # LUN failover (multi-attach): a non-zero key makes the instance + # register it on the LUN at open, and an active one (standby False) + # take the reservation, preempting the old holder (fence). Sent only + # when set: older data planes reject the parameters. + if reservation_key: + params["reservation_key"] = int(reservation_key) + if standby: + params["standby"] = True return self._request("bdev_alceml_create", **params) + + def alceml_get_role(self, name): + """LUN failover state of an alceml (role, state, epoch, keys, ...).""" + return self._request("alceml_get_role", name=name) def bdev_distrib_create(self, name, vuid, ndcs, npcs, num_blocks, block_size, jm_names, chunk_size, ha_comm_addrs=None, ha_inode_self=None, pba_page_size=2097152, diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 9cf6654bd6..ac729fb223 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -2061,7 +2061,7 @@ def _create_storage_device_stack(rpc_client, nvme, snode: StorageNode, after_res cache_size=cache_size, cache_eviction_threshold=cache_eviction_threshold, force_4k_atomic=getattr(cluster, 'atomic_4k', False), - **multi_attach.alceml_create_kwargs(nvme), + **multi_attach.alceml_create_kwargs(nvme, snode, cluster), ) if not ret: diff --git a/simplyblock_core/utils/nvme.py b/simplyblock_core/utils/nvme.py index 1ed6cce416..8b533e5e32 100644 --- a/simplyblock_core/utils/nvme.py +++ b/simplyblock_core/utils/nvme.py @@ -130,6 +130,12 @@ def build_nvme_connect_entry( # reconnect 2 s (docs/design/two-node-arbitration.md, client # requirements). Clusters without arbitration keep the defaults. keep_alive_tmo = constants.TWO_NODE_CLIENT_KEEP_ALIVE_TO + elif getattr(cluster, "multi_attach", False) is True: + # Multi-attach (1+0): a node lost without closing its connections + # leaves its own volumes' paths dead until the host's keep-alive + # notices (constants.MULTI_ATTACH_CLIENT_KEEP_ALIVE_TO). Regular + # clusters keep the defaults. + keep_alive_tmo = constants.MULTI_ATTACH_CLIENT_KEEP_ALIVE_TO client_data_nic_str = f"--host-iface={cluster.client_data_nic}" if cluster.client_data_nic else "" tls_str = host_auth_str = "" if host_entry: diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index c673383051..bee89881b7 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -950,3 +950,110 @@ def test_base_bdev_still_small_after_rescan_fails_on_that_node(self): b.spdk.bdev_aio_rescan = lambda name: True self.assertNotEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) + + +# ------------------------------------- fence by persistent reservation + +class TestFenceByReservation(_TakeoverCase): + """ultra #236: the takeover's alceml opens with the standby's key and + preempts the failed node's reservation (fence in ms); the array fence + follows off the I/O path. Without the reservation: array fence first.""" + + def _roles(self, node, *answers): + seq = list(answers) + node.spdk.alceml_get_role = lambda name: seq.pop(0) if len(seq) > 1 else seq[0] + + def _bed(self): + db, dev = _bed() + dev.lun_reservations = True + return db, dev + + def _wait_background_fence(self): + import time as _t + for _ in range(100): + if FakeArrayPlugin.calls: + return + _t.sleep(0.02) + + def test_reservation_fences_before_the_array(self): + db, _ = self._bed() + b = db.nodes["node-b"] + self._roles(b, {"role": "activating", "state": "act_reservation"}, + {"role": "active", "state": "active", "epoch": 2}) + order = [] + create = b.spdk.bdev_alceml_create + b.spdk.bdev_alceml_create = lambda *a, **k: (order.append(("alceml", FakeArrayPlugin.calls[:])), + create(*a, **k))[1] + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + # Nothing at the array before the alceml (whose open preempts). + self.assertEqual(order, [("alceml", [])]) + alceml = b.spdk.ops("bdev_alceml_create")[0] + self.assertEqual(alceml[2]["reservation_key"], multi_attach.reservation_key(b)) + self.assertEqual(alceml[2]["reservation_key"], 2 << 32) + self.assertNotIn("standby", alceml[2]) + self.assertEqual(self.dev(db).serving_node(), "node-b") + self.assertIn("node-a", self.dev(db).fenced_node_ids) + # The array fence follows in the background. + self._wait_background_fence() + self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-a")]) + + def test_failed_activation_falls_back_to_the_array_fence(self): + db, _ = self._bed() + b = db.nodes["node-b"] + self._roles(b, {"role": "standby", "state": "act_failed"}, + {"role": "active", "state": "active"}) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-a")]) + # The first instance was removed, the second one took the reservation. + self.assertEqual(len(b.spdk.ops("bdev_alceml_create")), 2) + self.assertEqual(len(b.spdk.ops("bdev_alceml_delete")), 1) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + def test_without_reservations_the_array_fences_first(self): + db, _ = _bed() + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + alceml = db.nodes["node-b"].spdk.ops("bdev_alceml_create")[0] + self.assertNotIn("reservation_key", alceml[2]) + + +class TestReservationKeys(unittest.TestCase): + + def setUp(self): + array.forget_plugin() + self.addCleanup(array.forget_plugin) + + def test_key_is_the_node_block_plus_one_in_the_upper_half(self): + n = _node("n", "10.0.0.9", 0) + self.assertEqual(multi_attach.reservation_key(n), 1 << 32) + n.cntlid_block = 41 + self.assertEqual(multi_attach.reservation_key(n), 42 << 32) + n.cntlid_block = -1 + self.assertEqual(multi_attach.reservation_key(n), 0) + + def test_kwargs_follow_the_device_record(self): + db, dev = _bed() + b = db.nodes["node-b"] + self.assertNotIn("reservation_key", multi_attach.alceml_create_kwargs(dev, b, db.cluster)) + dev.lun_reservations = True + kw = multi_attach.alceml_create_kwargs(dev, b, db.cluster) + self.assertEqual(kw["reservation_key"], 2 << 32) + self.assertEqual(kw["stuck_io_timeout_ms"], constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS) + dev.multi_attach = False + self.assertEqual(multi_attach.alceml_create_kwargs(dev, b, db.cluster), {}) + + def test_decided_with_the_array_capability_and_checksum(self): + class PR(FakeArrayPlugin): + def capabilities(self): + return dict(super().capabilities(), supports_reservations=True) + array.register_plugin("pr-test", PR) + self.addCleanup(array.unregister_plugin, "pr-test") + db, dev = _bed() + self.assertFalse(multi_attach.reservations_supported(db.cluster, dev)) # fake: no PR + db.cluster.array_plugin = "pr-test" + array.forget_plugin() + self.assertTrue(multi_attach.reservations_supported(db.cluster, dev)) + db.cluster.inline_checksum, dev.md_supported = True, False # method 2 + self.assertFalse(multi_attach.reservations_supported(db.cluster, dev)) + with patch.object(constants, "MULTI_ATTACH_RESERVATIONS", False): + db.cluster.inline_checksum = False + self.assertFalse(multi_attach.reservations_supported(db.cluster, dev)) diff --git a/tests/unit/test_two_node_c1_fixes.py b/tests/unit/test_two_node_c1_fixes.py index b135e03a36..fc4fd8f5fd 100644 --- a/tests/unit/test_two_node_c1_fixes.py +++ b/tests/unit/test_two_node_c1_fixes.py @@ -361,3 +361,15 @@ def test_connect_entry_tells_the_csi_driver_it_is_two_node(arbitrated): cluster=_Cluster(arbitrated), host_entry=None, host_nqn=None) assert entry.two_node is arbitrated assert entry.model_dump(by_alias=True)["two-node"] is arbitrated + + +def test_keep_alive_for_multi_attach_clusters(): + """1+0 multi-attach: a hard-reset node's own volumes waited 10-14 s for the + 8 s keep-alive (FTT1/FTT2 beds, 2026-11-20).""" + cl = _Cluster(False) + cl.multi_attach = True + entry = build_nvme_connect_entry(transport="tcp", ip="10.0.0.1", port=4420, nqn="nqn.x", + ctrl_loss_tmo=constants.LVOL_NVME_CONNECT_CTRL_LOSS_TMO, + cluster=cl, host_entry=None, host_nqn=None) + assert entry.keep_alive_tmo == constants.MULTI_ATTACH_CLIENT_KEEP_ALIVE_TO == 3 + assert "--keep-alive-tmo=3" in entry.connect From 826149c823c235f2420b64b5a5ad6cdd558af991 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 10:45:22 +0100 Subject: [PATCH 27/50] feat(credentials): KMS-backed credential store for external systems Groundwork for storage-array plugins (provision/resize/delete LUNs): plugins hold a credential reference and resolve it at call time; secrets never sit in FoundationDB in clear. - KMS: generic secret wrap/unwrap (wrap_secret/unwrap_secret) on all three backends, with a context binding (array identity + endpoint) and an embedded SHA-256 digest -- KMIP wraps with AES-CBC, so the digest is what catches tampering there. ensure_key_encryption_key never replaces an existing KEK (KMIP create was not idempotent). - Vault: read_kv_secret for customer-managed KV v2 secrets; optional credentials_cert_role so a separate Vault policy can cover credentials. - KMIP: get_secret_data for customer-managed SecretData objects. - simplyblock_core.credentials: CredentialProvider with managed refs (wrapped under a per-cluster KEK) and external refs (Vault KV path, KMIP SecretData UID), in-memory TTL cache, invalidate-on-auth-failure. - LocalKMS (no external KMS) only encodes, so credentials are refused unless the cluster opts in via allow_local_kms_credentials. - cluster set: parse bool attributes explicitly; bool("false") was True. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/cluster_ops.py | 13 +- simplyblock_core/credentials.py | 311 +++++++++++++ simplyblock_core/kms/__init__.py | 12 +- simplyblock_core/kms/_base.py | 87 ++++ simplyblock_core/kms/_fdb.py | 19 + simplyblock_core/kms/_hcp.py | 52 +++ simplyblock_core/kms/_kmip.py | 46 +- simplyblock_core/kms/_settings.py | 5 + simplyblock_core/models/cluster.py | 6 + tests/integration/kms/test_kmip_client.py | 66 +++ tests/unit/test_credentials.py | 516 ++++++++++++++++++++++ 11 files changed, 1128 insertions(+), 5 deletions(-) create mode 100644 simplyblock_core/credentials.py create mode 100644 tests/unit/test_credentials.py diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 0972bbbb5a..8413e2bdc5 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -4680,7 +4680,18 @@ def set_(cl_id, attr, value) -> bool: cluster.write_to_db() return True else: - value = cluster.get_attrs_map()[attr]['type'](value) + attr_type = cluster.get_attrs_map()[attr]['type'] + if attr_type is bool and isinstance(value, str): + # bool("false") is True; parse the CLI's string form explicitly. + lowered = value.strip().lower() + if lowered in ("1", "true", "yes", "on"): + value = True + elif lowered in ("0", "false", "no", "off"): + value = False + else: + raise ValueError(f"Not a boolean value for {attr}: {value!r}") + else: + value = attr_type(value) logger.info(f"Setting {attr} to {value}") if attr == "two_node_arbitration": set_two_node_arbitration(cl_id, value) diff --git a/simplyblock_core/credentials.py b/simplyblock_core/credentials.py new file mode 100644 index 0000000000..b65103acf2 --- /dev/null +++ b/simplyblock_core/credentials.py @@ -0,0 +1,311 @@ +"""Credentials for external systems (storage arrays, ...), kept out of the database. + +Plugins that drive an external system (provision/resize/delete a LUN on a +storage array, ...) never store or fetch secrets themselves. Their records +hold a :data:`CredentialRef`; at call time they ask a +:class:`CredentialProvider` to resolve it into a :class:`Credential`, which +exists only in memory. + +Two kinds of reference: + +*Managed* (:class:`ManagedCredentialRef`) + simplyblock received the secret (CLI/API) and keeps it wrapped under a + cluster KEK in the cluster's KMS: Vault Transit or a KMIP appliance. Only + the ciphertext is stored, inside the ref. Revoking the KEK in the KMS + makes every managed credential of the cluster unreadable at once. + +*External* (:class:`VaultKVCredentialRef`, :class:`KMIPSecretCredentialRef`) + The customer owns and rotates the secret in their own Vault KV v2 mount or + as a KMIP ``SecretData`` object; simplyblock stores only the pointer. + +Every resolution is bound to a *context* string naming what the credential is +for (e.g. ``"array//https://pure01.example:443"``). A managed credential +carries its context inside the wrapped payload and is refused for any other, +so rewriting an array's endpoint in the database cannot redirect the +credential to another host. An external secret may opt into the same check by +holding a ``_context`` field. + +With no external KMS configured the cluster falls back to ``LocalKMS``, which +only encodes. Managed credentials are then refused unless the cluster has +explicitly opted in via ``Cluster.allow_local_kms_credentials``. + +Resolved credentials are cached in memory for ``ttl_s``. On an authentication +failure from the external system the caller invalidates the ref and resolves +again once, which picks up a rotated external secret without a restart. +""" +from __future__ import annotations + +import json +import logging +import threading +import time +from collections.abc import Callable, Mapping +from typing import TYPE_CHECKING, Annotated, Literal, Union + +from pydantic import BaseModel, ConfigDict, Field, SecretStr, TypeAdapter + +from simplyblock_core.kms import ( + KMS, + HCPClient, + KMIPClient, + KMIPKMSSettings, + KMSException, + VaultKMSSettings, + create_kms_connection, +) + +if TYPE_CHECKING: + from simplyblock_core.models.cluster import Cluster + +logger = logging.getLogger(__name__) + +#: Field an external secret may hold to bind itself to one context. +CONTEXT_FIELD = "_context" + +Backend = Literal["vault", "kmip", "local"] + + +class CredentialError(Exception): + """A credential could not be stored or resolved. Never carries secret values.""" + + +class ManagedCredentialRef(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["managed"] = "managed" + #: The KMS backend that wrapped it; a different backend cannot unwrap it. + backend: Backend + kek_name: str = Field(min_length=1) + wrapped: str = Field(min_length=1) + + +class VaultKVCredentialRef(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["vault-kv"] = "vault-kv" + mount: str = Field(min_length=1) + path: str = Field(min_length=1) + #: Pin a KV version; ``None`` follows the latest (i.e. picks up rotation). + version: int | None = Field(default=None, ge=1) + + +class KMIPSecretCredentialRef(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["kmip-secret"] = "kmip-secret" + uid: str = Field(min_length=1) + #: ``None``: the object's value is a JSON object of string fields. + #: Set: the raw value (UTF-8) is that single field, e.g. ``"password"``. + field: str | None = Field(default=None, min_length=1) + + +CredentialRef = Annotated[ + Union[ManagedCredentialRef, VaultKVCredentialRef, KMIPSecretCredentialRef], + Field(discriminator="kind"), +] +CREDENTIAL_REF_ADAPTER: TypeAdapter[CredentialRef] = TypeAdapter(CredentialRef) + + +class Credential: + """Resolved secret fields, in memory only. ``repr`` shows field names only.""" + + __slots__ = ("_fields",) + + def __init__(self, fields: Mapping[str, SecretStr]): + self._fields = dict(fields) + + @property + def field_names(self) -> frozenset[str]: + return frozenset(self._fields) + + def get(self, name: str) -> SecretStr: + try: + return self._fields[name] + except KeyError: + raise CredentialError( + f"Credential has no field {name!r} (has: {', '.join(sorted(self._fields))})" + ) from None + + def get_optional(self, name: str) -> SecretStr | None: + return self._fields.get(name) + + def __repr__(self) -> str: + return f"Credential(fields={sorted(self._fields)})" + + +def kek_name_for(cluster_id: str) -> str: + """The cluster KEK that wraps managed credentials.""" + return f"credentials-{cluster_id}" + + +def _backend_of(cluster: Cluster) -> Backend: + settings = cluster.get_kms_settings() + if settings is None: + return "local" + if isinstance(settings, VaultKMSSettings): + return "vault" + if isinstance(settings, KMIPKMSSettings): + return "kmip" + raise CredentialError(f"Unsupported KMS settings: {type(settings).__name__}") + + +class CredentialProvider: + """Stores and resolves credentials for one cluster. Thread-safe.""" + + def __init__( + self, + cluster: Cluster, + *, + ttl_s: float = 300.0, + clock: Callable[[], float] = time.monotonic, + kms_factory: Callable[..., KMS] = create_kms_connection, + ): + self._cluster = cluster + self._ttl_s = ttl_s + self._clock = clock + self._kms_factory = kms_factory + self._lock = threading.Lock() + self._cache: dict[tuple[str, str], tuple[float, Credential]] = {} + + # -- KMS access --------------------------------------------------------- + + def _open_kms(self) -> KMS: + backend = _backend_of(self._cluster) + if backend == "local" and not self._cluster.allow_local_kms_credentials: + raise CredentialError( + "No external KMS (Vault or KMIP) is configured for this cluster. " + "Configure one, or explicitly opt in to keeping credentials in the " + "cluster database (effectively unencrypted) by setting " + "allow_local_kms_credentials on the cluster." + ) + cert_role = None + settings = self._cluster.get_kms_settings() + if isinstance(settings, VaultKMSSettings): + cert_role = settings.credentials_cert_role + try: + return self._kms_factory(self._cluster, cert_role=cert_role) + except KMSException as e: + raise CredentialError(f"Cannot connect to the KMS: {e.message}") from e + + # -- Store -------------------------------------------------------------- + + def store( + self, fields: Mapping[str, SecretStr | str], context: str, + ) -> ManagedCredentialRef: + """Wrap ``fields`` under the cluster's credentials KEK, bound to + ``context``. Persist the returned ref; the plaintext is not kept.""" + if not context: + raise CredentialError("A credential needs a non-empty context") + if not fields: + raise CredentialError("Refusing to store an empty credential") + secret_fields = { + name: value if isinstance(value, SecretStr) else SecretStr(value) + for name, value in fields.items() + } + backend = _backend_of(self._cluster) + kek_name = kek_name_for(self._cluster.get_id()) + try: + with self._open_kms() as kms: + kms.ensure_key_encryption_key(kek_name) + wrapped = kms.wrap_secret(kek_name, secret_fields, context) + except KMSException as e: + raise CredentialError(f"Storing credential failed: {e.message}") from e + logger.info("Stored managed credential (backend=%s, fields=%s, context=%s)", + backend, sorted(secret_fields), context) + return ManagedCredentialRef(backend=backend, kek_name=kek_name, wrapped=wrapped) + + # -- Resolve ------------------------------------------------------------ + + @staticmethod + def _cache_key(ref: CredentialRef, context: str) -> tuple[str, str]: + return ref.model_dump_json(), context + + def resolve(self, ref: CredentialRef | Mapping, context: str) -> Credential: + """Return the credential for ``ref``, bound to ``context``.""" + if isinstance(ref, Mapping): + ref = CREDENTIAL_REF_ADAPTER.validate_python(ref) + key = self._cache_key(ref, context) + now = self._clock() + with self._lock: + hit = self._cache.get(key) + if hit is not None and hit[0] > now: + return hit[1] + + credential = self._fetch(ref, context) + + with self._lock: + self._cache[key] = (self._clock() + self._ttl_s, credential) + return credential + + def invalidate(self, ref: CredentialRef | None = None) -> None: + """Drop cached resolutions of ``ref`` (all, if ``None``). Call after + the external system rejected the credential, then resolve once more.""" + with self._lock: + if ref is None: + self._cache.clear() + return + dumped = ref.model_dump_json() + for key in [k for k in self._cache if k[0] == dumped]: + del self._cache[key] + + def _fetch(self, ref: CredentialRef, context: str) -> Credential: + if isinstance(ref, ManagedCredentialRef): + backend = _backend_of(self._cluster) + if ref.backend != backend: + raise CredentialError( + f"Credential was stored with the {ref.backend} KMS backend, but the " + f"cluster now uses {backend}; store it again" + ) + try: + with self._open_kms() as kms: + if isinstance(ref, ManagedCredentialRef): + return Credential(kms.unwrap_secret(ref.kek_name, ref.wrapped, context)) + if isinstance(ref, VaultKVCredentialRef): + if not isinstance(kms, HCPClient): + raise CredentialError( + "A Vault KV credential needs the cluster KMS to be Vault") + fields = kms.read_kv_secret(ref.mount, ref.path, ref.version) + where = f"Vault {ref.mount}/{ref.path}" + else: + if not isinstance(kms, KMIPClient): + raise CredentialError( + "A KMIP secret credential needs the cluster KMS to be KMIP") + fields = self._parse_secret_data(kms.get_secret_data(ref.uid), ref) + where = f"KMIP object {ref.uid}" + except KMSException as e: + raise CredentialError(f"Resolving credential failed: {e.message}") from e + return Credential(self._check_external_context(fields, context, where)) + + @staticmethod + def _parse_secret_data( + raw: bytes, ref: KMIPSecretCredentialRef, + ) -> dict[str, SecretStr]: + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + raise CredentialError(f"KMIP object {ref.uid} is not UTF-8 text") from e + if ref.field is not None: + return {ref.field: SecretStr(text)} + try: + data = json.loads(text) + except ValueError: + data = None + if not isinstance(data, dict) or not data or not all( + isinstance(k, str) and isinstance(v, str) for k, v in data.items() + ): + raise CredentialError( + f"KMIP object {ref.uid} is not a JSON object of string fields; " + "set 'field' on the reference to use its raw value as one field" + ) + return {name: SecretStr(value) for name, value in data.items()} + + @staticmethod + def _check_external_context( + fields: dict[str, SecretStr], context: str, where: str, + ) -> dict[str, SecretStr]: + bound = fields.pop(CONTEXT_FIELD, None) + if bound is not None and bound.get_secret_value() != context: + raise CredentialError(f"Secret at {where} is bound to a different context") + if not fields: + raise CredentialError(f"Secret at {where} has no credential fields") + return fields diff --git a/simplyblock_core/kms/__init__.py b/simplyblock_core/kms/__init__.py index 95b424ffca..f366d00819 100644 --- a/simplyblock_core/kms/__init__.py +++ b/simplyblock_core/kms/__init__.py @@ -17,9 +17,12 @@ __all__ = [ "KMS", + "HCPClient", + "KMIPClient", "KMIPKMSSettings", "KMSException", "KMSSettings", + "LocalKMS", "VaultKMSSettings", "backup_dek_path", "backup_kek_name", @@ -55,7 +58,12 @@ def backup_kek_name(backup_id: str) -> str: return f"backup-{backup_id}" -def create_kms_connection(cluster: "Cluster") -> KMS: +def create_kms_connection(cluster: "Cluster", *, cert_role: str | None = None) -> KMS: + """Open the cluster's KMS backend. + + ``cert_role`` overrides the configured Vault cert-auth role; the other + backends ignore it. + """ kms_settings = cluster.get_kms_settings() if kms_settings is None: return LocalKMS(cluster) @@ -81,7 +89,7 @@ def create_kms_connection(cluster: "Cluster") -> KMS: settings.tls_key, transit_mount=kms_settings.transit_mount, kv_mount=kms_settings.kv_mount, - cert_role=kms_settings.cert_role, + cert_role=cert_role or kms_settings.cert_role, ) return KMIPClient( diff --git a/simplyblock_core/kms/_base.py b/simplyblock_core/kms/_base.py index 1bc6126b9a..93dcbfa4e5 100644 --- a/simplyblock_core/kms/_base.py +++ b/simplyblock_core/kms/_base.py @@ -1,11 +1,22 @@ from __future__ import annotations +import hashlib +import hmac +import json from abc import abstractmethod +from collections.abc import Mapping from contextlib import AbstractContextManager from types import TracebackType from pydantic import SecretStr +from ._exceptions import KMSException + +#: Version of the envelope ``wrap_secret`` produces. Bump on any change to the +#: payload layout; ``unwrap_secret`` refuses versions it does not know. +_SECRET_ENVELOPE_VERSION = 1 +_DIGEST_LEN = hashlib.sha256().digest_size + class KMS(AbstractContextManager): def __exit__( # Has to be defined to make the type-checker happy @@ -49,6 +60,82 @@ def create_key_encryption_key(self, name: str) -> None: ... @abstractmethod def delete_key_encryption_key(self, name: str) -> None: ... + def ensure_key_encryption_key(self, name: str) -> None: + """Create the KEK ``name`` unless it already exists. + + Must never replace an existing key: everything wrapped under it would + become unreadable. Backends whose ``create_key_encryption_key`` is not + already idempotent override this. + """ + self.create_key_encryption_key(name) + + # -- Opaque secrets (credentials for external systems) ------------------ + # + # Unlike data encryption keys these are not random key material but small + # sets of named string fields (username/password, API token, PEM client + # certificate, ...). Only the wrapped form is ever persisted. + + def _wrap_bytes(self, kek_name: str, plaintext: bytes) -> str: + """Encrypt ``plaintext`` under the KEK, returning a storable string.""" + raise NotImplementedError(f"{type(self).__name__} cannot wrap secrets") + + def _unwrap_bytes(self, kek_name: str, wrapped: str) -> bytes: + """Inverse of :meth:`_wrap_bytes`.""" + raise NotImplementedError(f"{type(self).__name__} cannot unwrap secrets") + + def wrap_secret( + self, kek_name: str, fields: Mapping[str, SecretStr], context: str, + ) -> str: + """Wrap ``fields`` under ``kek_name``, bound to ``context``. + + ``context`` names what the secret is *for* (e.g. the external system's + identity and endpoint). :meth:`unwrap_secret` refuses to return the + secret for any other context, so a record whose endpoint was rewritten + in the database cannot be used to send the credential elsewhere. + + A SHA-256 digest of the payload is wrapped together with it. Vault + Transit (AES-GCM) already authenticates its ciphertext; the KMIP + backend wraps with AES-CBC (see ``_kmip.py``), which does not, and the + embedded digest turns blind tampering with that ciphertext into a + refusal instead of a silently altered secret or context. + """ + if not fields: + raise KMSException("Refusing to wrap an empty secret") + payload = json.dumps( + { + "v": _SECRET_ENVELOPE_VERSION, + "ctx": context, + "fields": {name: value.get_secret_value() for name, value in fields.items()}, + }, + sort_keys=True, separators=(",", ":"), + ).encode() + return self._wrap_bytes(kek_name, hashlib.sha256(payload).digest() + payload) + + def unwrap_secret( + self, kek_name: str, wrapped: str, context: str, + ) -> dict[str, SecretStr]: + """Inverse of :meth:`wrap_secret`; ``context`` must match exactly.""" + blob = self._unwrap_bytes(kek_name, wrapped) + digest, payload = blob[:_DIGEST_LEN], blob[_DIGEST_LEN:] + if len(digest) != _DIGEST_LEN or not hmac.compare_digest( + digest, hashlib.sha256(payload).digest(), + ): + raise KMSException("Wrapped secret failed its integrity check") + try: + data = json.loads(payload) + except ValueError as e: + raise KMSException("Wrapped secret is not valid") from e + if not isinstance(data, dict) or data.get("v") != _SECRET_ENVELOPE_VERSION: + raise KMSException("Wrapped secret has an unsupported format") + if not hmac.compare_digest(str(data.get("ctx", "")).encode(), context.encode()): + raise KMSException("Wrapped secret is bound to a different context") + fields = data.get("fields") + if not isinstance(fields, dict) or not fields or not all( + isinstance(k, str) and isinstance(v, str) for k, v in fields.items() + ): + raise KMSException("Wrapped secret is not valid") + return {name: SecretStr(value) for name, value in fields.items()} + # Generic secrets: a flat ``{name: value}`` dict stored at ``path`` (array # credentials, see simplyblock_core.array). Values go in as ``SecretStr`` # (plain strings are wrapped) and come back as ``SecretStr``; only the diff --git a/simplyblock_core/kms/_fdb.py b/simplyblock_core/kms/_fdb.py index 99bc1f362f..f2b94c07e0 100644 --- a/simplyblock_core/kms/_fdb.py +++ b/simplyblock_core/kms/_fdb.py @@ -1,3 +1,5 @@ +import base64 +import binascii import json from typing import TYPE_CHECKING @@ -79,3 +81,20 @@ def get_secret(self, path: str) -> dict: def delete_secret(self, path: str) -> None: self._kv_store.clear(self._secret_key(path)) + + # There is no KEK here: the "wrapped" form is only encoded, not encrypted, + # so a secret wrapped by LocalKMS is effectively plaintext in FoundationDB. + # Credential storage gates this behind an explicit per-cluster opt-in + # (see simplyblock_core.credentials). + _LOCAL_PREFIX = "local:" + + def _wrap_bytes(self, kek_name: str, plaintext: bytes) -> str: + return self._LOCAL_PREFIX + base64.b64encode(plaintext).decode() + + def _unwrap_bytes(self, kek_name: str, wrapped: str) -> bytes: + if not wrapped.startswith(self._LOCAL_PREFIX): + raise KMSException("Wrapped secret was not produced by the local KMS") + try: + return base64.b64decode(wrapped[len(self._LOCAL_PREFIX):], validate=True) + except binascii.Error as e: + raise KMSException("Wrapped secret is not valid") from e diff --git a/simplyblock_core/kms/_hcp.py b/simplyblock_core/kms/_hcp.py index eada641254..593b22b918 100644 --- a/simplyblock_core/kms/_hcp.py +++ b/simplyblock_core/kms/_hcp.py @@ -170,3 +170,55 @@ def delete_secret(self, path: str) -> None: return except hvac.exceptions.VaultError as e: raise KMSException("Request failed") from e + + def ensure_key_encryption_key(self, name: str) -> None: + try: + self.client.secrets.transit.read_key(name=name, mount_point=self.transit_mount) + return + except hvac.exceptions.InvalidPath: + pass + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + self.create_key_encryption_key(name) + + def _wrap_bytes(self, kek_name: str, plaintext: bytes) -> str: + try: + return self.client.secrets.transit.encrypt_data( + name=kek_name, plaintext=base64.b64encode(plaintext).decode(), + mount_point=self.transit_mount, + )['data']['ciphertext'] + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + + def _unwrap_bytes(self, kek_name: str, wrapped: str) -> bytes: + try: + plaintext_b64 = self.client.secrets.transit.decrypt_data( + name=kek_name, ciphertext=wrapped, mount_point=self.transit_mount, + )['data']['plaintext'] + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + return base64.b64decode(plaintext_b64) + + def read_kv_secret( + self, mount: str, path: str, version: int | None = None, + ) -> dict[str, SecretStr]: + """Read a customer-managed KV v2 secret as named string fields. + + Errors name the path and field names, never values. + """ + try: + data = self.client.secrets.kv.v2.read_secret_version( + path=path, mount_point=mount, version=version, + raise_on_deleted_version=True, + )['data']['data'] + except hvac.exceptions.InvalidPath as e: + raise KMSException(f"No secret at {mount}/{path}") from e + except hvac.exceptions.VaultError as e: + raise KMSException("Request failed") from e + if not isinstance(data, dict) or not data: + raise KMSException(f"Secret at {mount}/{path} is empty") + bad = sorted(name for name, value in data.items() if not isinstance(value, str)) + if bad: + raise KMSException( + f"Secret at {mount}/{path} has non-string fields: {', '.join(bad)}") + return {name: SecretStr(value) for name, value in data.items()} diff --git a/simplyblock_core/kms/_kmip.py b/simplyblock_core/kms/_kmip.py index cf1d472300..ffa0bb3d4a 100644 --- a/simplyblock_core/kms/_kmip.py +++ b/simplyblock_core/kms/_kmip.py @@ -3,6 +3,7 @@ from kmip.core import enums from kmip.pie import exceptions as kmip_exceptions +from kmip.pie import objects as kmip_objects from kmip.pie.client import ProxyKmipClient from pydantic import SecretStr @@ -133,9 +134,12 @@ def delete_key_encryption_key(self, name: str) -> None: kv_store.clear(self._kek_key(name)) def _encrypt(self, kek_uid: str, plaintext_hex: SecretStr) -> dict: + return self._encrypt_bytes(kek_uid, bytes.fromhex(plaintext_hex.get_secret_value())) + + def _encrypt_bytes(self, kek_uid: str, plaintext: bytes) -> dict: try: ciphertext, iv_counter_nonce = self.client.encrypt( - bytes.fromhex(plaintext_hex.get_secret_value()), + plaintext, uid=kek_uid, cryptographic_parameters=dict(_CRYPTOGRAPHIC_PARAMETERS), ) @@ -147,6 +151,9 @@ def _encrypt(self, kek_uid: str, plaintext_hex: SecretStr) -> dict: } def _decrypt(self, kek_uid: str, wrapped: dict) -> SecretStr: + return SecretStr(self._decrypt_bytes(kek_uid, wrapped).hex()) + + def _decrypt_bytes(self, kek_uid: str, wrapped: dict) -> bytes: try: plaintext = self.client.decrypt( bytes.fromhex(wrapped["ciphertext"]), @@ -156,7 +163,7 @@ def _decrypt(self, kek_uid: str, wrapped: dict) -> SecretStr: ) except _KMIP_ERRORS as e: raise KMSException("Request failed") from e - return SecretStr(plaintext.hex()) + return plaintext def create_data_encryption_keys(self, path: str, kek_name: str) -> None: self.import_data_encryption_keys( @@ -200,3 +207,38 @@ def get_secret(self, path: str) -> dict: def delete_secret(self, path: str) -> None: raise KMSException(self._NO_SECRETS) + + def ensure_key_encryption_key(self, name: str) -> None: + # create_key_encryption_key always mints a new appliance key and + # repoints the name at it, which would orphan everything already + # wrapped under the old one. + if self._kv_store().get(self._kek_key(name)): + return + self.create_key_encryption_key(name) + + def _wrap_bytes(self, kek_name: str, plaintext: bytes) -> str: + return json.dumps(self._encrypt_bytes(self._kek_uid(kek_name), plaintext)) + + def _unwrap_bytes(self, kek_name: str, wrapped: str) -> bytes: + try: + parsed = json.loads(wrapped) + if not isinstance(parsed, dict) or "ciphertext" not in parsed: + raise ValueError("not a wrapped KMIP record") + return self._decrypt_bytes(self._kek_uid(kek_name), parsed) + except ValueError as e: # bad JSON, or bytes.fromhex on a corrupted record + raise KMSException("Wrapped secret is not valid") from e + + def get_secret_data(self, uid: str) -> bytes: + """Fetch a customer-managed KMIP ``SecretData`` object's raw value. + + Support for ``SecretData`` varies between appliances (some only manage + keys); a server that refuses it surfaces as ``KMSException``. + """ + try: + obj = self.client.get(uid) + except _KMIP_ERRORS as e: + raise KMSException(f"Could not fetch KMIP object {uid}") from e + if not isinstance(obj, kmip_objects.SecretData): + raise KMSException( + f"KMIP object {uid} is a {type(obj).__name__}, not SecretData") + return bytes(obj.value) diff --git a/simplyblock_core/kms/_settings.py b/simplyblock_core/kms/_settings.py index ca1c92d992..fa9ebf21d9 100644 --- a/simplyblock_core/kms/_settings.py +++ b/simplyblock_core/kms/_settings.py @@ -29,6 +29,11 @@ class VaultKMSSettings(BaseModel): transit_mount: str = "simplyblock/transit" kv_mount: str = "simplyblock/kv" cert_role: str = "simplyblock-webappapi" + #: Vault cert-auth role used for credentials of external systems (storage + #: arrays, ...) instead of ``cert_role``, so a Vault policy can grant those + #: paths separately from volume encryption keys. ``None`` reuses + #: ``cert_role``. + credentials_cert_role: str | None = None #: Same convention as BackupLocation._serialize_endpoint: the field owns #: its own JSON-safety, so plain model_dump() is enough here too. diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 283a3fda6b..ae0c5f40a5 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -463,6 +463,12 @@ def is_topology_owned(self) -> bool: # migration for every create/update. Empty dict = no external KMS (LocalKMS). kms_settings: dict = default_factory(dict) hashicorp_vault_settings: HashicorpVaultSettings | None = None + # Explicit opt-in to keep credentials for external systems (storage + # arrays, ...) with no external KMS configured. LocalKMS only encodes, so + # such credentials would sit effectively in plaintext in FoundationDB; + # without this flag simplyblock_core.credentials refuses to store or read + # them unless a Vault or KMIP backend is configured. + allow_local_kms_credentials: bool = False # Single-node-expansion resumability cursor. Empty dict means no expansion # is in flight. Populated/advanced/cleared via the helpers in diff --git a/tests/integration/kms/test_kmip_client.py b/tests/integration/kms/test_kmip_client.py index e37470362a..e817737659 100644 --- a/tests/integration/kms/test_kmip_client.py +++ b/tests/integration/kms/test_kmip_client.py @@ -114,3 +114,69 @@ def test_rekey_moves_keys_to_a_new_kek_and_path(kmip_client): def test_delete_key_encryption_key_of_an_unknown_name_is_a_no_op(kmip_client): kmip_client.delete_key_encryption_key(_unique("kek")) + + +# -- Opaque secrets (credentials for external systems) ---------------------- + +_CTX = "array/a1/https://pure01.example:443" + + +def test_wrap_secret_round_trip(kmip_client): + kek_name = _unique("kek") + kmip_client.ensure_key_encryption_key(kek_name) + try: + wrapped = kmip_client.wrap_secret( + kek_name, {"username": SecretStr("u"), "password": SecretStr("p")}, _CTX) + fields = kmip_client.unwrap_secret(kek_name, wrapped, _CTX) + assert {k: v.get_secret_value() for k, v in fields.items()} == { + "username": "u", "password": "p"} + with pytest.raises(KMSException, match="different context"): + kmip_client.unwrap_secret(kek_name, wrapped, "array/a1/https://evil.example:443") + finally: + kmip_client.delete_key_encryption_key(kek_name) + + +def test_ensure_key_encryption_key_keeps_existing_key(kmip_client): + kek_name = _unique("kek") + kmip_client.ensure_key_encryption_key(kek_name) + try: + wrapped = kmip_client.wrap_secret(kek_name, {"a": SecretStr("b")}, _CTX) + kmip_client.ensure_key_encryption_key(kek_name) + assert kmip_client.unwrap_secret(kek_name, wrapped, _CTX)["a"].get_secret_value() == "b" + finally: + kmip_client.delete_key_encryption_key(kek_name) + + +@pytest.fixture() +def kmip_client_no_fdb(pykmip_server): + """SecretData Get never touches FoundationDB.""" + client = KMIPClient( + pykmip_server.server_url, + pykmip_server.ca, pykmip_server.certificate, pykmip_server.key, + ) + with client: + yield client + + +def test_get_secret_data_returns_customer_secret(kmip_client_no_fdb): + from kmip.core import enums + from kmip.pie import objects + + value = b'{"username": "u", "password": "p"}' + uid = kmip_client_no_fdb.client.register( + objects.SecretData(value, enums.SecretDataType.PASSWORD)) + try: + assert kmip_client_no_fdb.get_secret_data(uid) == value + finally: + kmip_client_no_fdb.client.destroy(uid) + + +def test_get_secret_data_refuses_a_key_object(kmip_client_no_fdb): + from kmip.core import enums + + uid = kmip_client_no_fdb.client.create(enums.CryptographicAlgorithm.AES, 256) + try: + with pytest.raises(KMSException, match="not SecretData"): + kmip_client_no_fdb.get_secret_data(uid) + finally: + kmip_client_no_fdb.client.destroy(uid) diff --git a/tests/unit/test_credentials.py b/tests/unit/test_credentials.py new file mode 100644 index 0000000000..5761f9920e --- /dev/null +++ b/tests/unit/test_credentials.py @@ -0,0 +1,516 @@ +"""Credentials for external systems: KMS secret envelope + CredentialProvider.""" +import base64 +import json +from unittest.mock import MagicMock + +import hvac.exceptions +import pytest +from kmip.core import enums +from kmip.pie import objects as kmip_objects +from pydantic import SecretStr + +from simplyblock_core.credentials import ( + CONTEXT_FIELD, + Credential, + CredentialError, + CredentialProvider, + KMIPSecretCredentialRef, + ManagedCredentialRef, + VaultKVCredentialRef, + kek_name_for, +) +from simplyblock_core.kms import KMS, HCPClient, KMIPClient, KMSException, LocalKMS +from simplyblock_core.models.cluster import Cluster + +CTX = "array/a1/https://pure01.example:443" + + +class FakeKMS(KMS): + """Reversible, keyed 'encryption' so tests can see wrong-key failures.""" + + def __init__(self): + self.keks: set[str] = set() + self.created: list[str] = [] + + def create_data_encryption_keys(self, path, kek_name): ... + def import_data_encryption_keys(self, path, kek_name, keys): ... + def get_data_encryption_keys(self, path, kek_name): ... + def delete_data_encryption_keys(self, path): ... + + def create_key_encryption_key(self, name): + self.created.append(name) + self.keks.add(name) + + def delete_key_encryption_key(self, name): + self.keks.discard(name) + + def ensure_key_encryption_key(self, name): + if name not in self.keks: + self.create_key_encryption_key(name) + + def _wrap_bytes(self, kek_name, plaintext): + if kek_name not in self.keks: + raise KMSException("no such KEK") + return base64.b64encode(kek_name.encode() + b"|" + plaintext).decode() + + def _unwrap_bytes(self, kek_name, wrapped): + if kek_name not in self.keks: + raise KMSException("no such KEK") + prefix, _, plaintext = base64.b64decode(wrapped).partition(b"|") + if prefix != kek_name.encode(): + raise KMSException("wrong KEK") + return plaintext + + +def _fields(**kw): + return {k: SecretStr(v) for k, v in kw.items()} + + +def _plain(fields): + return {k: v.get_secret_value() for k, v in fields.items()} + + +# -- Envelope (KMS.wrap_secret / unwrap_secret) ----------------------------- + +class TestEnvelope: + + def setup_method(self): + self.kms = FakeKMS() + self.kms.create_key_encryption_key("k") + + def test_round_trip(self): + wrapped = self.kms.wrap_secret("k", _fields(username="u", password="p"), CTX) + assert _plain(self.kms.unwrap_secret("k", wrapped, CTX)) == { + "username": "u", "password": "p"} + + def test_other_context_is_refused(self): + wrapped = self.kms.wrap_secret("k", _fields(password="p"), CTX) + with pytest.raises(KMSException, match="different context"): + self.kms.unwrap_secret("k", wrapped, "array/a1/https://evil.example:443") + + def test_tampered_payload_is_refused(self): + wrapped = self.kms.wrap_secret("k", _fields(password="p"), CTX) + raw = bytearray(base64.b64decode(wrapped)) + raw[-3] ^= 0x01 # inside the JSON payload, after the digest + with pytest.raises(KMSException, match="integrity"): + self.kms.unwrap_secret("k", base64.b64encode(bytes(raw)).decode(), CTX) + + def test_truncated_blob_is_refused(self): + wrapped = base64.b64encode(b"k|short").decode() + with pytest.raises(KMSException, match="integrity"): + self.kms.unwrap_secret("k", wrapped, CTX) + + def test_unknown_version_is_refused(self): + import hashlib + payload = json.dumps({"v": 99, "ctx": CTX, "fields": {"a": "b"}}).encode() + wrapped = self.kms._wrap_bytes("k", hashlib.sha256(payload).digest() + payload) + with pytest.raises(KMSException, match="unsupported format"): + self.kms.unwrap_secret("k", wrapped, CTX) + + def test_empty_secret_is_refused(self): + with pytest.raises(KMSException): + self.kms.wrap_secret("k", {}, CTX) + + def test_backend_without_wrap_support_says_so(self): + class NoWrap(FakeKMS): + _wrap_bytes = KMS._wrap_bytes + with pytest.raises(NotImplementedError, match="cannot wrap"): + NoWrap().wrap_secret("k", _fields(a="b"), CTX) + + +# -- Backends --------------------------------------------------------------- + +class TestLocalKMSWrapping: + + def _kms(self): + return object.__new__(LocalKMS) # no FDB needed for wrap/unwrap + + def test_round_trip(self): + kms = self._kms() + wrapped = kms.wrap_secret("ignored", _fields(token="t"), CTX) + assert wrapped.startswith("local:") + assert _plain(kms.unwrap_secret("ignored", wrapped, CTX)) == {"token": "t"} + + def test_foreign_ciphertext_is_refused(self): + with pytest.raises(KMSException, match="local KMS"): + self._kms().unwrap_secret("k", "vault:v1:abc", CTX) + + def test_corrupt_encoding_is_refused(self): + with pytest.raises(KMSException): + self._kms().unwrap_secret("k", "local:!!!", CTX) + + +class _FakeKVStore: + def __init__(self): + self.data = {} + + def get(self, key): + return self.data.get(key) + + def set(self, key, value): + self.data[key] = value + + def clear(self, key): + self.data.pop(key, None) + + +@pytest.fixture() +def kmip(monkeypatch): + client = object.__new__(KMIPClient) + client.client = MagicMock() + kv = _FakeKVStore() + monkeypatch.setattr(KMIPClient, "_kv_store", staticmethod(lambda: kv)) + uids = iter(f"uid-{i}" for i in range(100)) + client.client.create.side_effect = lambda *a, **k: next(uids) + client.client.encrypt.side_effect = lambda data, **k: (bytes(b ^ 0x5A for b in data), b"\x01" * 16) + client.client.decrypt.side_effect = lambda data, **k: bytes(b ^ 0x5A for b in data) + return client, kv + + +class TestKMIPWrapping: + + def test_round_trip(self, kmip): + client, _ = kmip + client.ensure_key_encryption_key("k") + wrapped = client.wrap_secret("k", _fields(password="p"), CTX) + assert json.loads(wrapped)["iv"] == "01" * 16 + assert _plain(client.unwrap_secret("k", wrapped, CTX)) == {"password": "p"} + + def test_ensure_does_not_replace_existing_kek(self, kmip): + client, kv = kmip + client.ensure_key_encryption_key("k") + client.ensure_key_encryption_key("k") + assert client.client.create.call_count == 1 + assert kv.get(b"kms/kmip/kek/k") == b"uid-0" + + def test_cbc_bit_flip_is_caught_by_digest(self, kmip): + client, _ = kmip + client.ensure_key_encryption_key("k") + record = json.loads(client.wrap_secret("k", _fields(password="p"), CTX)) + ct = bytearray.fromhex(record["ciphertext"]) + ct[40] ^= 0x01 + record["ciphertext"] = ct.hex() + with pytest.raises(KMSException, match="integrity"): + client.unwrap_secret("k", json.dumps(record), CTX) + + def test_garbage_record_is_refused(self, kmip): + client, _ = kmip + client.ensure_key_encryption_key("k") + for bad in ("not json", "[]", '{"ciphertext": "zz"}'): + with pytest.raises(KMSException, match="not valid"): + client.unwrap_secret("k", bad, CTX) + + def test_get_secret_data(self, kmip): + client, _ = kmip + client.client.get.return_value = kmip_objects.SecretData( + b'{"password": "p"}', enums.SecretDataType.PASSWORD) + assert client.get_secret_data("u1") == b'{"password": "p"}' + + def test_get_secret_data_refuses_keys(self, kmip): + client, _ = kmip + client.client.get.return_value = kmip_objects.SymmetricKey( + enums.CryptographicAlgorithm.AES, 128, b"\x00" * 16) + with pytest.raises(KMSException, match="not SecretData"): + client.get_secret_data("u1") + + +@pytest.fixture() +def vault(): + client = object.__new__(HCPClient) + client.client = MagicMock() + client.transit_mount = "t" + client.kv_mount = "kv" + return client + + +class TestVault: + + def test_ensure_creates_only_when_missing(self, vault): + transit = vault.client.secrets.transit + transit.read_key.side_effect = hvac.exceptions.InvalidPath() + vault.ensure_key_encryption_key("k") + transit.create_key.assert_called_once() + + transit.reset_mock() + transit.read_key.side_effect = None + vault.ensure_key_encryption_key("k") + transit.create_key.assert_not_called() + + def test_wrap_uses_transit(self, vault): + transit = vault.client.secrets.transit + store = {} + transit.encrypt_data.side_effect = lambda name, plaintext, mount_point: ( + store.setdefault("pt", plaintext), {"data": {"ciphertext": "vault:v1:x"}})[1] + transit.decrypt_data.side_effect = lambda name, ciphertext, mount_point: { + "data": {"plaintext": store["pt"]}} + wrapped = vault.wrap_secret("k", _fields(password="p"), CTX) + assert wrapped == "vault:v1:x" + assert _plain(vault.unwrap_secret("k", wrapped, CTX)) == {"password": "p"} + + def test_read_kv_secret(self, vault): + vault.client.secrets.kv.v2.read_secret_version.return_value = { + "data": {"data": {"username": "u", "password": "p"}}} + assert _plain(vault.read_kv_secret("arrays", "pure01")) == { + "username": "u", "password": "p"} + + def test_read_kv_secret_refuses_non_string_fields_without_values(self, vault): + vault.client.secrets.kv.v2.read_secret_version.return_value = { + "data": {"data": {"password": "hunter2", "port": 443}}} + with pytest.raises(KMSException) as e: + vault.read_kv_secret("arrays", "pure01") + assert "port" in e.value.message and "hunter2" not in e.value.message + + def test_read_kv_secret_missing(self, vault): + vault.client.secrets.kv.v2.read_secret_version.side_effect = hvac.exceptions.InvalidPath() + with pytest.raises(KMSException, match="No secret"): + vault.read_kv_secret("arrays", "nope") + + +# -- CredentialProvider ----------------------------------------------------- + +def _cluster(kms_settings=None, allow_local=False): + cluster = Cluster() + cluster.uuid = "c1" + cluster.set_kms_settings(kms_settings) + cluster.allow_local_kms_credentials = allow_local + return cluster + + +VAULT = {"provider": "vault", "base_url": "https://vault.example:8200"} +KMIP = {"provider": "kmip", "server_url": "kmip.example:5696"} + + +class Clock: + def __init__(self): + self.t = 1000.0 + + def __call__(self): + return self.t + + +def _provider(cluster, kms, clock=None): + factory = MagicMock(return_value=kms) + provider = CredentialProvider(cluster, kms_factory=factory, clock=clock or Clock()) + return provider, factory + + +class TestOptIn: + + def test_local_kms_is_refused_without_opt_in(self): + provider, factory = _provider(_cluster(), FakeKMS()) + with pytest.raises(CredentialError, match="opt in"): + provider.store({"password": "p"}, CTX) + factory.assert_not_called() + + def test_local_kms_refuses_resolve_after_opt_in_is_withdrawn(self): + cluster = _cluster(allow_local=True) + provider, _ = _provider(cluster, FakeKMS()) + ref = provider.store({"password": "p"}, CTX) + cluster.allow_local_kms_credentials = False + with pytest.raises(CredentialError, match="opt in"): + CredentialProvider(cluster, kms_factory=MagicMock(return_value=FakeKMS())).resolve(ref, CTX) + + def test_local_kms_works_with_opt_in(self): + provider, _ = _provider(_cluster(allow_local=True), FakeKMS()) + ref = provider.store({"password": "p"}, CTX) + assert ref.backend == "local" + assert provider.resolve(ref, CTX).get("password").get_secret_value() == "p" + + def test_cluster_default_is_no_opt_in(self): + assert Cluster().allow_local_kms_credentials is False + + +class TestManaged: + + def test_store_and_resolve_round_trip(self): + kms = FakeKMS() + provider, factory = _provider(_cluster(VAULT), kms) + ref = provider.store({"username": "u", "password": SecretStr("hunter2")}, CTX) + assert ref == ManagedCredentialRef( + backend="vault", kek_name=kek_name_for("c1"), wrapped=ref.wrapped) + assert kms.created == ["credentials-c1"] + cred = provider.resolve(ref.model_dump(), CTX) # dict form, as stored in FDB + assert cred.get("password").get_secret_value() == "hunter2" + assert "hunter2" not in repr(cred) + + def test_kek_is_reused(self): + kms = FakeKMS() + provider, _ = _provider(_cluster(VAULT), kms) + provider.store({"a": "1"}, CTX) + provider.store({"a": "2"}, CTX) + assert kms.created == ["credentials-c1"] + + def test_other_context_is_refused(self): + provider, _ = _provider(_cluster(KMIP), FakeKMS()) + ref = provider.store({"password": "p"}, CTX) + with pytest.raises(CredentialError, match="different context"): + provider.resolve(ref, "array/a1/https://evil.example:443") + + def test_backend_change_is_reported(self): + kms = FakeKMS() + provider, _ = _provider(_cluster(VAULT), kms) + ref = provider.store({"password": "p"}, CTX) + provider2, _ = _provider(_cluster(KMIP), kms) + with pytest.raises(CredentialError, match="stored with the vault"): + provider2.resolve(ref, CTX) + + def test_empty_inputs_are_refused(self): + provider, _ = _provider(_cluster(VAULT), FakeKMS()) + with pytest.raises(CredentialError): + provider.store({}, CTX) + with pytest.raises(CredentialError): + provider.store({"a": "b"}, "") + + def test_credentials_cert_role_is_used(self): + settings = dict(VAULT, credentials_cert_role="simplyblock-credentials") + provider, factory = _provider(_cluster(settings), FakeKMS()) + provider.store({"a": "b"}, CTX) + assert factory.call_args.kwargs == {"cert_role": "simplyblock-credentials"} + + def test_kms_connection_failure_is_wrapped(self): + provider = CredentialProvider( + _cluster(VAULT), kms_factory=MagicMock(side_effect=KMSException("Authentication failed"))) + with pytest.raises(CredentialError, match="Authentication failed"): + provider.store({"a": "b"}, CTX) + + +class TestCache: + + def test_ttl_and_invalidate(self): + kms = FakeKMS() + clock = Clock() + provider, factory = _provider(_cluster(VAULT), kms, clock) + ref = provider.store({"a": "b"}, CTX) + factory.reset_mock() + + provider.resolve(ref, CTX) + provider.resolve(ref, CTX) + assert factory.call_count == 1 + + clock.t += 301 + provider.resolve(ref, CTX) + assert factory.call_count == 2 + + provider.invalidate(ref) + provider.resolve(ref, CTX) + assert factory.call_count == 3 + + provider.invalidate() + provider.resolve(ref, CTX) + assert factory.call_count == 4 + + def test_cache_is_per_context(self): + provider, _ = _provider(_cluster(VAULT), FakeKMS()) + ref = provider.store({"a": "b"}, CTX) + provider.resolve(ref, CTX) + with pytest.raises(CredentialError): + provider.resolve(ref, "other") + + +class TestExternal: + + def _vault_kms(self, data): + kms = MagicMock(spec=HCPClient) + kms.__enter__.return_value = kms + kms.read_kv_secret.return_value = {k: SecretStr(v) for k, v in data.items()} + return kms + + def _kmip_kms(self, raw): + kms = MagicMock(spec=KMIPClient) + kms.__enter__.return_value = kms + kms.get_secret_data.return_value = raw + return kms + + def test_vault_kv(self): + kms = self._vault_kms({"username": "u", "password": "p"}) + provider, _ = _provider(_cluster(VAULT), kms) + ref = VaultKVCredentialRef(mount="customer-kv", path="arrays/pure01", version=3) + cred = provider.resolve(ref, CTX) + assert cred.field_names == {"username", "password"} + kms.read_kv_secret.assert_called_once_with("customer-kv", "arrays/pure01", 3) + + def test_vault_kv_context_field(self): + provider, _ = _provider( + _cluster(VAULT), self._vault_kms({"password": "p", CONTEXT_FIELD: CTX})) + assert provider.resolve( + VaultKVCredentialRef(mount="m", path="p"), CTX).field_names == {"password"} + + provider, _ = _provider( + _cluster(VAULT), self._vault_kms({"password": "p", CONTEXT_FIELD: "other"})) + with pytest.raises(CredentialError, match="different context"): + provider.resolve(VaultKVCredentialRef(mount="m", path="p"), CTX) + + def test_vault_kv_needs_vault(self): + provider, _ = _provider(_cluster(KMIP), self._kmip_kms(b"{}")) + with pytest.raises(CredentialError, match="needs the cluster KMS to be Vault"): + provider.resolve(VaultKVCredentialRef(mount="m", path="p"), CTX) + + def test_kmip_json_object(self): + provider, _ = _provider( + _cluster(KMIP), self._kmip_kms(b'{"username": "u", "password": "p"}')) + cred = provider.resolve({"kind": "kmip-secret", "uid": "42"}, CTX) + assert cred.get("password").get_secret_value() == "p" + + def test_kmip_raw_value_as_field(self): + provider, _ = _provider(_cluster(KMIP), self._kmip_kms(b"s3cr3t")) + cred = provider.resolve(KMIPSecretCredentialRef(uid="42", field="api_token"), CTX) + assert cred.get("api_token").get_secret_value() == "s3cr3t" + + def test_kmip_raw_value_without_field_is_refused(self): + provider, _ = _provider(_cluster(KMIP), self._kmip_kms(b"s3cr3t")) + with pytest.raises(CredentialError, match="set 'field'") as e: + provider.resolve(KMIPSecretCredentialRef(uid="42"), CTX) + assert "s3cr3t" not in str(e.value) + + def test_kmip_needs_kmip(self): + provider, _ = _provider(_cluster(VAULT), self._vault_kms({"a": "b"})) + with pytest.raises(CredentialError, match="needs the cluster KMS to be KMIP"): + provider.resolve(KMIPSecretCredentialRef(uid="42"), CTX) + + def test_external_refs_refused_under_local_kms_without_opt_in(self): + provider, _ = _provider(_cluster(), FakeKMS()) + with pytest.raises(CredentialError, match="opt in"): + provider.resolve(VaultKVCredentialRef(mount="m", path="p"), CTX) + + +def test_credential_missing_field_names_fields_not_values(): + cred = Credential(_fields(username="u", password="hunter2")) + with pytest.raises(CredentialError) as e: + cred.get("token") + assert "password" in str(e.value) and "hunter2" not in str(e.value) + + +# -- cluster set: bool parsing --------------------------------------------- + +@pytest.mark.parametrize("raw, expected", [ + ("true", True), ("True", True), ("1", True), ("yes", True), + ("false", False), ("False", False), ("0", False), ("off", False), +]) +def test_cluster_set_parses_bools(monkeypatch, raw, expected): + from simplyblock_core import cluster_ops + + cluster = Cluster() + cluster.allow_local_kms_credentials = not expected + cluster.write_to_db = MagicMock() + db = MagicMock() + db.get_cluster_by_id.return_value = cluster + monkeypatch.setattr(cluster_ops, "db_controller", db) + + cluster_ops.set_("c1", "allow_local_kms_credentials", raw) + + assert cluster.allow_local_kms_credentials is expected + cluster.write_to_db.assert_called_once() + + +def test_cluster_set_rejects_non_bool(monkeypatch): + from simplyblock_core import cluster_ops + + cluster = Cluster() + cluster.write_to_db = MagicMock() + db = MagicMock() + db.get_cluster_by_id.return_value = cluster + monkeypatch.setattr(cluster_ops, "db_controller", db) + + with pytest.raises(ValueError): + cluster_ops.set_("c1", "allow_local_kms_credentials", "maybe") + cluster.write_to_db.assert_not_called() From 00b47151e262c1484dd307b68bbe90fae6f11731 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 13:56:51 +0100 Subject: [PATCH 28/50] feat(storage_arrays): adapter framework, fake-array test harness, Pure FlashArray adapter ArrayClient (credential resolution + re-auth retry, sb- naming, size rounding, idempotent create/map/unmap/delete, ownership + WWN checks), RestSession (TLS/CA bundle, retries, error mapping), lazy vendor registry, in-process vendor fakes + HTTPS mock server, shared contract suite with a live tier (SB_ARRAY_LIVE_CONFIG). Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/storage_arrays/__init__.py | 44 +++ .../storage_arrays/adapters/__init__.py | 1 + .../storage_arrays/adapters/pure.py | 224 +++++++++++ simplyblock_core/storage_arrays/base.py | 354 ++++++++++++++++++ simplyblock_core/storage_arrays/client.py | 268 +++++++++++++ simplyblock_core/storage_arrays/http.py | 198 ++++++++++ simplyblock_core/storage_arrays/registry.py | 61 +++ .../storage_arrays/testing/__init__.py | 314 ++++++++++++++++ .../storage_arrays/testing/__main__.py | 48 +++ .../storage_arrays/testing/pure.py | 270 +++++++++++++ tests/unit/storage_arrays/__init__.py | 0 tests/unit/storage_arrays/conftest.py | 115 ++++++ tests/unit/storage_arrays/test_contract.py | 270 +++++++++++++ 13 files changed, 2167 insertions(+) create mode 100644 simplyblock_core/storage_arrays/__init__.py create mode 100644 simplyblock_core/storage_arrays/adapters/__init__.py create mode 100644 simplyblock_core/storage_arrays/adapters/pure.py create mode 100644 simplyblock_core/storage_arrays/base.py create mode 100644 simplyblock_core/storage_arrays/client.py create mode 100644 simplyblock_core/storage_arrays/http.py create mode 100644 simplyblock_core/storage_arrays/registry.py create mode 100644 simplyblock_core/storage_arrays/testing/__init__.py create mode 100644 simplyblock_core/storage_arrays/testing/__main__.py create mode 100644 simplyblock_core/storage_arrays/testing/pure.py create mode 100644 tests/unit/storage_arrays/__init__.py create mode 100644 tests/unit/storage_arrays/conftest.py create mode 100644 tests/unit/storage_arrays/test_contract.py diff --git a/simplyblock_core/storage_arrays/__init__.py b/simplyblock_core/storage_arrays/__init__.py new file mode 100644 index 0000000000..d0ada602c8 --- /dev/null +++ b/simplyblock_core/storage_arrays/__init__.py @@ -0,0 +1,44 @@ +"""External storage arrays: vendor adapters behind one contract. + +Entry point: :class:`simplyblock_core.storage_arrays.client.ArrayClient`. +""" +from .base import ( + ArrayAdapter, + ArrayAuthError, + ArrayConfig, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + Capabilities, + HostSpec, + Initiator, + Mapping, + Protocol, + Volume, +) +from .registry import ADAPTERS, ALIASES, adapter_class, canonical_vendor + +__all__ = [ + "ADAPTERS", + "ALIASES", + "ArrayAdapter", + "ArrayAuthError", + "ArrayConfig", + "ArrayConflictError", + "ArrayError", + "ArrayInfo", + "ArrayNotFoundError", + "ArrayRequestError", + "ArrayUnsupportedError", + "Capabilities", + "HostSpec", + "Initiator", + "Mapping", + "Protocol", + "Volume", + "adapter_class", + "canonical_vendor", +] diff --git a/simplyblock_core/storage_arrays/adapters/__init__.py b/simplyblock_core/storage_arrays/adapters/__init__.py new file mode 100644 index 0000000000..9763b23894 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/__init__.py @@ -0,0 +1 @@ +"""Vendor adapters; see ..registry for the vendor keys.""" diff --git a/simplyblock_core/storage_arrays/adapters/pure.py b/simplyblock_core/storage_arrays/adapters/pure.py new file mode 100644 index 0000000000..5c2ed4864e --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/pure.py @@ -0,0 +1,224 @@ +"""Pure Storage FlashArray, REST API 2.x (Purity//FA 6.x and later). + +Auth: an API token (``api_token`` credential field) exchanged for a session +token via ``POST /api/2.x/login`` (``api-token`` header -> ``x-auth-token``). + +Objects are addressed by name (hosts) and id (volumes). Pure "destroys" +volumes into a recycle bin (recoverable for the eradication delay, 24 h by +default) before eradication; ``eradicate=true`` removes them at once. + +Volume identity: SCSI ``naa.624a9370``; NVMe NGUID +``eui.0024a937``. +""" +from __future__ import annotations + +from typing import Any + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import json_of + +_SUPPORTED_MAX = (2, 36) # newest 2.x minor this adapter was written against +_MIN_VERSION = (2, 4) + + +def _ver(v: str) -> tuple[int, int]: + major, _, minor = v.partition(".") + return int(major), int(minor or 0) + + +def pure_wwn(serial: str) -> str: + return "naa.624a9370" + serial.lower() + + +def pure_nguid(serial: str) -> str: + s = serial.lower() + return "eui.00" + s[:14] + "24a937" + s[14:] + + +class PureFlashArrayAdapter(ArrayAdapter): + vendor = "pure-flasharray" + display_name = "Pure Storage FlashArray" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC, Protocol.NVME_TCP, + Protocol.NVME_RDMA, Protocol.NVME_FC}), + size_granularity_bytes=512, + min_size_bytes=1 << 20, + max_name_len=63, + ) + credential_fields = (frozenset({"api_token"}),) + option_docs = ( + OptionDoc("eradicate", "Eradicate deleted volumes immediately instead of " + "leaving them recoverable for the eradication delay.", default=False), + OptionDoc("api_version", "Pin the REST 2.x version (e.g. '2.21'); " + "default: newest supported by both sides."), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._api = "" + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + version = self.option("api_version") + if not version: + offered = json_of(self.session.get("/api/api_version")).get("version", []) + usable = [v for v in offered + if v.startswith("2.") and _MIN_VERSION <= _ver(v) <= _SUPPORTED_MAX] + if not usable: + raise ArrayRequestError( + f"FlashArray offers no REST 2.x version in " + f"{'.'.join(map(str, _MIN_VERSION))}..{'.'.join(map(str, _SUPPORTED_MAX))}: " + f"{offered}") + version = max(usable, key=_ver) + self._api = f"/api/{version}" + self.session.headers.pop("x-auth-token", None) + resp = self.session.post(f"{self._api}/login", + headers={"api-token": self.secret("api_token")}) + token = resp.headers.get("x-auth-token") + if not token: + raise ArrayRequestError("FlashArray login returned no x-auth-token") + self.session.headers["x-auth-token"] = token + + def close(self) -> None: + if self._api and "x-auth-token" in self.session.headers: + try: + self.session.post(f"{self._api}/logout") + except Exception: + pass + + # -- request helpers ----------------------------------------------------- + + def _items(self, method: str, path: str, *, params=None, json=None, + missing_ok: bool = False) -> list[dict[str, Any]]: + resp = self.session.request(method, f"{self._api}{path}", params=params, json=json, + ok=(400,)) + body = json_of(resp) or {} + if resp.status_code == 400: + messages = " ".join( + str(e.get("message", "")) for e in body.get("errors", [])) + lowered = messages.lower() + if "does not exist" in lowered or "not found" in lowered: + if missing_ok: + return [] + raise ArrayNotFoundError(f"{method} {path}: {messages}") + if "already" in lowered or "in use" in lowered or "exists" in lowered: + raise ArrayConflictError(f"{method} {path}: {messages}") + raise ArrayRequestError(f"{method} {path}: {messages}", status=400) + return list(body.get("items", [])) + + @staticmethod + def _volume(item: dict[str, Any]) -> Volume: + serial = item.get("serial") or "" + return Volume( + id=item["id"], name=item["name"], size_bytes=int(item["provisioned"]), + wwn=pure_wwn(serial) if serial else None, + nguid=pure_nguid(serial) if serial else None, + ) + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + arr = self._items("GET", "/arrays")[0] + model = "" + try: + controllers = self._items("GET", "/controllers") + model = next((c.get("model", "") for c in controllers if c.get("model")), "") + except ArrayRequestError: + pass + return ArrayInfo(vendor=self.vendor, model=model, + version=f"{arr.get('os', 'Purity//FA')} {arr.get('version', '')}".strip(), + serial=arr.get("id", "")) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + items = self._items("POST", "/volumes", params={"names": name}, + json={"provisioned": size_bytes}) + return self._volume(items[0]) + + def _live(self, items: list[dict[str, Any]]) -> Volume | None: + live = [i for i in items if not i.get("destroyed")] + return self._volume(live[0]) if live else None + + def get_volume(self, volume_id: str) -> Volume | None: + return self._live(self._items("GET", "/volumes", params={"ids": volume_id}, + missing_ok=True)) + + def find_volume(self, name: str) -> Volume | None: + return self._live(self._items("GET", "/volumes", params={"names": name}, + missing_ok=True)) + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + items = self._items("PATCH", "/volumes", params={"ids": volume_id}, + json={"provisioned": size_bytes}) + return self._volume(items[0]) + + def delete_volume(self, volume_id: str) -> None: + self._items("PATCH", "/volumes", params={"ids": volume_id}, json={"destroyed": True}) + if self.option("eradicate"): + self._items("DELETE", "/volumes", params={"ids": volume_id}) + + def _host(self, name: str) -> dict[str, Any] | None: + items = self._items("GET", "/hosts", params={"names": name}, missing_ok=True) + return items[0] if items else None + + def ensure_host(self, spec: HostSpec) -> str: + wanted = { + "iqns": spec.addresses(Protocol.ISCSI), + "wwns": [a.upper() for a in spec.addresses(Protocol.FC)], + "nqns": spec.addresses(Protocol.NVME_TCP, Protocol.NVME_RDMA, Protocol.NVME_FC), + } + wanted = {k: v for k, v in wanted.items() if v} + host = self._host(spec.name) + if host is None: + self._items("POST", "/hosts", params={"names": spec.name}, json=wanted) + return spec.name + add: dict[str, list[str]] = {} + for key, values in wanted.items(): + have = {str(v).lower() for v in host.get(key, [])} + missing = [v for v in values if v.lower() not in have] + if missing: + add[f"add_{key}"] = missing + if add: + self._items("PATCH", "/hosts", params={"names": spec.name}, json=add) + return spec.name + + def delete_host(self, host_id: str) -> None: + self._items("DELETE", "/hosts", params={"names": host_id}, missing_ok=True) + + def _connections(self, volume_id: str, host_id: str | None = None) -> list[Mapping]: + params: dict[str, str] = {"volume_ids": volume_id} + if host_id is not None: + params["host_names"] = host_id + items = self._items("GET", "/connections", params=params, missing_ok=True) + return [Mapping(volume_id=i["volume"]["id"], host_id=i["host"]["name"], + lun=i.get("lun")) for i in items if i.get("host")] + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + existing = self._connections(volume_id, host_id) + if existing: + return existing[0] + items = self._items("POST", "/connections", + params={"host_names": host_id, "volume_ids": volume_id}) + i = items[0] + return Mapping(volume_id=i["volume"]["id"], host_id=i["host"]["name"], lun=i.get("lun")) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + if not self._connections(volume_id, host_id): + return + self._items("DELETE", "/connections", + params={"host_names": host_id, "volume_ids": volume_id}, missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return self._connections(volume_id) diff --git a/simplyblock_core/storage_arrays/base.py b/simplyblock_core/storage_arrays/base.py new file mode 100644 index 0000000000..0619d402e6 --- /dev/null +++ b/simplyblock_core/storage_arrays/base.py @@ -0,0 +1,354 @@ +"""Vendor-neutral model and adapter contract for external storage arrays. + +An adapter translates five operations -- volume create/resize/delete, host +(initiator group) management and volume-to-host mapping -- into one vendor's +management API. Everything vendor-neutral (credential resolution, +re-authentication, size rounding, idempotence, delete/resize safety checks) +lives in :class:`simplyblock_core.storage_arrays.client.ArrayClient`, so +adapters stay thin and are tested against one shared contract suite +(``tests/unit/storage_arrays/test_contract.py``). +""" +from __future__ import annotations + +import re +from abc import ABC, abstractmethod +from collections.abc import Mapping as MappingABC +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, ClassVar + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from simplyblock_core.credentials import Credential + +# -- Errors ----------------------------------------------------------------- + + +class ArrayError(Exception): + """Base class. Messages never contain credential values.""" + + +class ArrayAuthError(ArrayError): + """The array rejected the credential or the session expired.""" + + +class ArrayNotFoundError(ArrayError): + """The addressed object does not exist.""" + + +class ArrayConflictError(ArrayError): + """The request conflicts with array state (name taken, initiator owned + by another host, volume still mapped, ...).""" + + +class ArrayUnsupportedError(ArrayError): + """The adapter or array does not support the requested operation.""" + + +class ArrayRequestError(ArrayError): + """Any other failed request.""" + + def __init__(self, message: str, status: int | None = None): + super().__init__(message) + self.status = status + + +# -- Model ------------------------------------------------------------------ + + +class Protocol(str, Enum): + ISCSI = "iscsi" + FC = "fc" + NVME_TCP = "nvme-tcp" + NVME_RDMA = "nvme-rdma" + NVME_FC = "nvme-fc" + #: Dell PowerFlex native client (SDC); the initiator address is the SDC GUID. + SDC = "sdc" + + @property + def is_nvme(self) -> bool: + return self in (Protocol.NVME_TCP, Protocol.NVME_RDMA, Protocol.NVME_FC) + + @property + def is_scsi(self) -> bool: + return self in (Protocol.ISCSI, Protocol.FC) + + +_WWPN_RE = re.compile(r"^[0-9a-f]{16}$") + + +def normalize_wwpn(value: str) -> str: + """``21:00:00:24:FF:4C:1A:2B`` / ``0x2100...`` -> ``21000024ff4c1a2b``.""" + v = value.strip().lower().removeprefix("0x").replace(":", "").replace("-", "") + if not _WWPN_RE.match(v): + raise ValueError(f"Not a WWPN: {value!r}") + return v + + +def colon_wwpn(value: str) -> str: + """``21000024ff4c1a2b`` -> ``21:00:00:24:ff:4c:1a:2b``.""" + v = normalize_wwpn(value) + return ":".join(v[i:i + 2] for i in range(0, 16, 2)) + + +class Initiator(BaseModel): + """One host port: an iSCSI IQN, an FC WWPN, an NVMe host NQN or an SDC GUID.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + protocol: Protocol + address: str = Field(min_length=1) + + @field_validator("address") + @classmethod + def _strip(cls, v: str) -> str: + return v.strip() + + def normalized(self) -> str: + if self.protocol in (Protocol.FC, Protocol.NVME_FC): + return normalize_wwpn(self.address) + if self.protocol == Protocol.ISCSI: + return self.address.lower() + return self.address + + +class HostSpec(BaseModel): + """A host as the array should know it. All initiators share one protocol + family (SCSI or NVMe); a host using both is registered twice.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str = Field(min_length=1, max_length=255) + initiators: tuple[Initiator, ...] = Field(min_length=1) + + @field_validator("initiators") + @classmethod + def _one_family(cls, v: tuple[Initiator, ...]) -> tuple[Initiator, ...]: + protocols = {i.protocol for i in v} + if len(protocols) > 1 and not ( + all(p.is_scsi for p in protocols) or all(p.is_nvme for p in protocols) + ): + raise ValueError("A host's initiators must all be SCSI or all be NVMe") + return v + + @property + def protocols(self) -> frozenset[Protocol]: + return frozenset(i.protocol for i in self.initiators) + + def addresses(self, *protocols: Protocol) -> list[str]: + return [i.normalized() for i in self.initiators if i.protocol in protocols] + + +class Volume(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + #: The array's own handle for the volume (id, uuid, device number, name). + id: str + name: str + size_bytes: int + #: SCSI identity as the host sees it, ``naa.`` (lowercase). This is + #: what matches the array volume to ``/dev/disk/by-id`` on the host. + wwn: str | None = None + #: NVMe identity, ``eui.<32 hex>`` (NGUID) where the array exposes one. + nguid: str | None = None + + @property + def identifier(self) -> str: + """Stable identity used for safety checks: WWN, else NGUID, else id.""" + return self.wwn or self.nguid or f"id.{self.id}" + + +class Mapping(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + volume_id: str + host_id: str + #: SCSI LUN or NVMe namespace ID as presented to the host; ``None`` where + #: the protocol has no such number (PowerFlex SDC). + lun: int | None = None + + +class ArrayInfo(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + vendor: str + model: str = "" + version: str = "" + serial: str = "" + + +class ArrayConfig(BaseModel): + """One registered array. Holds no secret: ``credential_ref`` is resolved + through :class:`simplyblock_core.credentials.CredentialProvider`.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + id: str = Field(min_length=1) + vendor: str = Field(min_length=1) + #: Management endpoint, e.g. ``https://pure01.example.com`` or + #: ``https://unisphere.example.com:8443``. Path prefixes are kept. + endpoint: str = Field(min_length=1) + credential_ref: dict[str, Any] + #: PEM CA bundle (or pinned self-signed certificate) for the endpoint. + ca_bundle: str | None = None + #: Must stay True outside of labs; ``False`` disables TLS verification. + verify_tls: bool = True + timeout_s: float = Field(default=60.0, gt=0) + #: Vendor-specific settings (pool, SVM, CPG, storage group, ...); each + #: adapter documents what it reads in ``option_docs``. + options: dict[str, Any] = Field(default_factory=dict) + + @field_validator("endpoint") + @classmethod + def _https_only(cls, v: str) -> str: + v = v.strip().rstrip("/") + if not v.lower().startswith("https://"): + raise ValueError("Array endpoint must be https://") + return v + + +@dataclass(frozen=True) +class Capabilities: + protocols: frozenset[Protocol] + #: Volume sizes are rounded up to a multiple of this. + size_granularity_bytes: int = 512 + min_size_bytes: int = 1 << 20 + #: Longest volume / host name the array accepts. + max_name_len: int = 63 + #: Whether resize can only grow (true for every supported array today). + grow_only: bool = True + + +@dataclass +class OptionDoc: + name: str + help: str + required: bool = False + default: Any = None + + +@dataclass +class AdapterContext: + """What an adapter is constructed with.""" + + config: ArrayConfig + credential: Credential + session: Any # simplyblock_core.storage_arrays.http.RestSession + extra: dict[str, Any] = field(default_factory=dict) + + +class ArrayAdapter(ABC): + """One vendor's implementation of the contract. + + Rules every adapter follows (the contract suite checks them): + + * ``create_volume`` receives a name that is already unique and short + enough, and a size already rounded to ``capabilities``. + * ``get_volume``/``find_volume`` return ``None`` for a missing volume; + every other method raises :class:`ArrayNotFoundError` for one. + * ``ensure_host`` is idempotent: an existing host of that name gets any + missing initiators added; an initiator registered to a *different* host + raises :class:`ArrayConflictError`. + * ``map_volume`` is idempotent and returns the existing mapping if there is + one; ``unmap_volume`` of an absent mapping is a no-op. + * ``delete_volume`` is only called with no mappings left. + * Errors never include credential values. + """ + + vendor: ClassVar[str] + display_name: ClassVar[str] + capabilities: ClassVar[Capabilities] + #: Accepted credential shapes, as alternatives of required field names, + #: e.g. ``({"api_token"}, {"username", "password"})``. + credential_fields: ClassVar[tuple[frozenset[str], ...]] = ( + frozenset({"username", "password"}), + ) + option_docs: ClassVar[tuple[OptionDoc, ...]] = () + + def __init__(self, ctx: AdapterContext): + self.config = ctx.config + self.credential = ctx.credential + self.session = ctx.session + self.options: MappingABC[str, Any] = ctx.config.options + self._check_credential_shape() + self._check_options() + + # -- helpers ------------------------------------------------------------ + + def _check_credential_shape(self) -> None: + have = self.credential.field_names + if not any(shape <= have for shape in self.credential_fields): + wanted = " or ".join("+".join(sorted(s)) for s in self.credential_fields) + raise ArrayAuthError( + f"{self.display_name} needs credential fields {wanted}; " + f"got {', '.join(sorted(have)) or 'none'}") + + def _check_options(self) -> None: + missing = [o.name for o in self.option_docs if o.required and o.name not in self.options] + if missing: + raise ArrayError(f"{self.display_name} array {self.config.id} needs options: " + + ", ".join(missing)) + known = {o.name for o in self.option_docs} + unknown = sorted(set(self.options) - known) + if unknown: + raise ArrayError(f"Unknown {self.display_name} options: {', '.join(unknown)}") + + def option(self, name: str) -> Any: + if name in self.options: + return self.options[name] + for o in self.option_docs: + if o.name == name: + return o.default + raise KeyError(name) + + def secret(self, name: str) -> str: + return self.credential.get(name).get_secret_value() + + def has_secret(self, name: str) -> bool: + return name in self.credential.field_names + + # -- lifecycle ---------------------------------------------------------- + + def login(self) -> None: + """Establish a session. Called lazily by the client and again after + an :class:`ArrayAuthError`. Basic-auth adapters may leave this empty.""" + + def close(self) -> None: + """Release the session (best effort, never raises).""" + + # -- contract ----------------------------------------------------------- + + @abstractmethod + def get_info(self) -> ArrayInfo: ... + + @abstractmethod + def create_volume(self, name: str, size_bytes: int) -> Volume: ... + + @abstractmethod + def get_volume(self, volume_id: str) -> Volume | None: ... + + @abstractmethod + def find_volume(self, name: str) -> Volume | None: ... + + @abstractmethod + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: ... + + @abstractmethod + def delete_volume(self, volume_id: str) -> None: ... + + @abstractmethod + def ensure_host(self, spec: HostSpec) -> str: + """Return the host id.""" + + @abstractmethod + def delete_host(self, host_id: str) -> None: + """Delete a host that has no mappings left (no-op if absent).""" + + @abstractmethod + def map_volume(self, volume_id: str, host_id: str) -> Mapping: ... + + @abstractmethod + def unmap_volume(self, volume_id: str, host_id: str) -> None: ... + + @abstractmethod + def list_mappings(self, volume_id: str) -> list[Mapping]: ... diff --git a/simplyblock_core/storage_arrays/client.py b/simplyblock_core/storage_arrays/client.py new file mode 100644 index 0000000000..e5bb928eab --- /dev/null +++ b/simplyblock_core/storage_arrays/client.py @@ -0,0 +1,268 @@ +"""Vendor-neutral front end for array adapters. + +:class:`ArrayClient` is what the rest of simplyblock calls. It + +* resolves the array's credential through the cluster's + :class:`~simplyblock_core.credentials.CredentialProvider`, bound to the + array's id and endpoint (so a rewritten endpoint cannot receive it); +* builds the adapter lazily and, when the array rejects the session or the + credential, invalidates the cached credential, logs in again and retries + the operation once (this is how rotated secrets are picked up); +* names volumes ``sb-`` from the simplyblock volume UUID, so names are + unique, fit every array's length limit, and mark ownership; +* rounds sizes up to the array's granularity; +* makes create/map/unmap/delete idempotent so an interrupted task can be + re-run; +* refuses to resize, unmap or delete a volume that is not simplyblock-owned + (name prefix) or whose WWN/NGUID differs from the one simplyblock recorded. +""" +from __future__ import annotations + +import logging +import threading +import uuid as uuid_mod +from collections.abc import Callable +from typing import Any, Self, TypeVar +from urllib.parse import urlsplit + +from requests.adapters import BaseAdapter + +from simplyblock_core.credentials import ( + CREDENTIAL_REF_ADAPTER, + Credential, + CredentialProvider, +) + +from .base import ( + AdapterContext, + ArrayAdapter, + ArrayAuthError, + ArrayConfig, + ArrayConflictError, + ArrayError, + ArrayInfo, + HostSpec, + Mapping, + Volume, +) +from .http import RestSession +from .registry import adapter_class, canonical_vendor + +logger = logging.getLogger(__name__) + +MANAGED_PREFIX = "sb-" +_NAME_HEX_LEN = 24 # 96 bits of the volume UUID: ample, and fits 31-char limits + +T = TypeVar("T") + + +def credential_context(config: ArrayConfig) -> str: + """The context an array's credential is bound to: id + scheme/host/port.""" + parts = urlsplit(config.endpoint) + port = parts.port or 443 + return f"array/{config.id}/{parts.scheme}://{(parts.hostname or '').lower()}:{port}" + + +def managed_name(volume_uuid: str, max_len: int, prefix: str = MANAGED_PREFIX) -> str: + hex_id = uuid_mod.UUID(str(volume_uuid)).hex + return (prefix + hex_id[:_NAME_HEX_LEN])[:max_len] + + +def round_size(size_bytes: int, granularity: int, minimum: int) -> int: + size = max(int(size_bytes), minimum) + return -(-size // granularity) * granularity + + +class ArrayClient: + """Thread-safe: operations on one array are serialized.""" + + def __init__( + self, + config: ArrayConfig, + credentials: CredentialProvider, + *, + transport: BaseAdapter | None = None, + session_kwargs: dict[str, Any] | None = None, + ): + self.config = config + self.vendor = canonical_vendor(config.vendor) + self.adapter_cls = adapter_class(self.vendor) + self._credentials = credentials + self._ref = CREDENTIAL_REF_ADAPTER.validate_python(config.credential_ref) + self._context = credential_context(config) + self._transport = transport + self._session_kwargs = session_kwargs or {} + self._adapter: ArrayAdapter | None = None + self._lock = threading.RLock() + + @property + def capabilities(self): + return self.adapter_cls.capabilities + + # -- adapter lifecycle ---------------------------------------------------- + + def _build(self) -> ArrayAdapter: + credential: Credential = self._credentials.resolve(self._ref, self._context) + session = RestSession.for_config(self.config, transport=self._transport, + **self._session_kwargs) + adapter = self.adapter_cls(AdapterContext(self.config, credential, session)) + try: + adapter.login() + except BaseException: + adapter.close() + session.close() + raise + return adapter + + def _drop(self) -> None: + if self._adapter is not None: + try: + self._adapter.close() + except Exception: # pragma: no cover - best effort + logger.debug("adapter close failed", exc_info=True) + try: + self._adapter.session.close() + except Exception: # pragma: no cover + pass + self._adapter = None + + def close(self) -> None: + with self._lock: + self._drop() + + def __enter__(self) -> Self: + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + def _call(self, op: str, fn: Callable[[ArrayAdapter], T]) -> T: + with self._lock: + for attempt in (1, 2): + try: + if self._adapter is None: + self._adapter = self._build() + return fn(self._adapter) + except ArrayAuthError: + self._drop() + if attempt == 2: + raise + logger.info("Array %s rejected the session during %s; re-authenticating", + self.config.id, op) + self._credentials.invalidate(self._ref) + raise AssertionError("unreachable") # pragma: no cover + + # -- checks ----------------------------------------------------------------- + + def _owned(self, vol: Volume, expected_identifier: str | None, action: str) -> None: + if not vol.name.startswith(MANAGED_PREFIX): + raise ArrayConflictError( + f"Refusing to {action} volume {vol.name!r} on array {self.config.id}: " + "not created by simplyblock") + if expected_identifier is not None and vol.identifier.lower() != expected_identifier.lower(): + raise ArrayConflictError( + f"Refusing to {action} volume {vol.name!r} on array {self.config.id}: " + f"identifier {vol.identifier} does not match recorded {expected_identifier}") + + # -- operations ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + return self._call("get_info", lambda a: a.get_info()) + + def volume_name(self, volume_uuid: str) -> str: + return managed_name(volume_uuid, self.capabilities.max_name_len) + + def create_volume(self, volume_uuid: str, size_bytes: int) -> Volume: + """Create (or, on re-run, return) the array volume for a simplyblock + volume. An existing volume of that name smaller than requested is grown.""" + name = self.volume_name(volume_uuid) + caps = self.capabilities + size = round_size(size_bytes, caps.size_granularity_bytes, caps.min_size_bytes) + + def op(a: ArrayAdapter) -> Volume: + existing = a.find_volume(name) + if existing is not None: + if existing.size_bytes < size: + return a.resize_volume(existing.id, size) + return existing + vol = a.create_volume(name, size) + logger.info("Array %s: created volume %s (%s, %d bytes)", + self.config.id, vol.name, vol.identifier, vol.size_bytes) + return vol + + return self._call("create_volume", op) + + def get_volume(self, volume_id: str) -> Volume | None: + return self._call("get_volume", lambda a: a.get_volume(volume_id)) + + def find_volume(self, volume_uuid: str) -> Volume | None: + name = self.volume_name(volume_uuid) + return self._call("find_volume", lambda a: a.find_volume(name)) + + def resize_volume(self, volume_id: str, size_bytes: int, + expected_identifier: str | None = None) -> Volume: + caps = self.capabilities + size = round_size(size_bytes, caps.size_granularity_bytes, caps.min_size_bytes) + + def op(a: ArrayAdapter) -> Volume: + vol = a.get_volume(volume_id) + if vol is None: + raise ArrayError(f"Volume {volume_id} not found on array {self.config.id}") + self._owned(vol, expected_identifier, "resize") + if size == vol.size_bytes: + return vol + if size < vol.size_bytes and caps.grow_only: + raise ArrayConflictError( + f"Refusing to shrink volume {vol.name} from {vol.size_bytes} to {size} bytes") + new = a.resize_volume(vol.id, size) + logger.info("Array %s: resized volume %s %d -> %d bytes", + self.config.id, vol.name, vol.size_bytes, new.size_bytes) + return new + + return self._call("resize_volume", op) + + def delete_volume(self, volume_id: str, expected_identifier: str | None = None) -> bool: + """Unmap from every host and delete. Returns False if already gone.""" + def op(a: ArrayAdapter) -> bool: + vol = a.get_volume(volume_id) + if vol is None: + return False + self._owned(vol, expected_identifier, "delete") + for m in a.list_mappings(vol.id): + a.unmap_volume(vol.id, m.host_id) + a.delete_volume(vol.id) + logger.info("Array %s: deleted volume %s (%s)", self.config.id, vol.name, vol.identifier) + return True + + return self._call("delete_volume", op) + + def ensure_host(self, spec: HostSpec) -> str: + unsupported = spec.protocols - self.capabilities.protocols + if unsupported: + raise ArrayError( + f"{self.adapter_cls.display_name} adapter does not support " + + ", ".join(sorted(p.value for p in unsupported))) + if len(spec.name) > self.capabilities.max_name_len: + raise ArrayError(f"Host name {spec.name!r} exceeds " + f"{self.capabilities.max_name_len} characters") + return self._call("ensure_host", lambda a: a.ensure_host(spec)) + + def delete_host(self, host_id: str) -> None: + self._call("delete_host", lambda a: a.delete_host(host_id)) + + def map_volume(self, volume_id: str, host_id: str, + expected_identifier: str | None = None) -> Mapping: + def op(a: ArrayAdapter) -> Mapping: + vol = a.get_volume(volume_id) + if vol is None: + raise ArrayError(f"Volume {volume_id} not found on array {self.config.id}") + self._owned(vol, expected_identifier, "map") + return a.map_volume(vol.id, host_id) + + return self._call("map_volume", op) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + self._call("unmap_volume", lambda a: a.unmap_volume(volume_id, host_id)) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return self._call("list_mappings", lambda a: a.list_mappings(volume_id)) diff --git a/simplyblock_core/storage_arrays/http.py b/simplyblock_core/storage_arrays/http.py new file mode 100644 index 0000000000..71c29fdc30 --- /dev/null +++ b/simplyblock_core/storage_arrays/http.py @@ -0,0 +1,198 @@ +"""Small HTTPS/JSON session shared by all array adapters. + +Adapters never build ``requests`` sessions themselves, so TLS verification, +timeouts, retries and error mapping are uniform, and tests can swap the +transport for an in-process fake (see ``storage_arrays.testing``). +""" +from __future__ import annotations + +import logging +import os +import tempfile +import time +from collections.abc import Callable, Iterable +from typing import Any + +import requests +from requests.adapters import BaseAdapter + +from .base import ( + ArrayAuthError, + ArrayConfig, + ArrayConflictError, + ArrayNotFoundError, + ArrayRequestError, +) + +logger = logging.getLogger(__name__) + +_IDEMPOTENT = frozenset({"GET", "HEAD", "OPTIONS"}) +_RETRY_STATUSES = frozenset({502, 503, 504}) +_MAX_ERROR_BODY = 600 + + +class RestSession: + """``requests`` wrapper bound to one array endpoint. + + * ``verify``: ``True`` (system CAs), a CA bundle path, or ``False``. + * Non-2xx responses raise: 401/403 -> :class:`ArrayAuthError`, 404 -> + :class:`ArrayNotFoundError`, 409 -> :class:`ArrayConflictError`, other -> + :class:`ArrayRequestError`. Pass ``ok=`` to accept extra statuses. + * Idempotent requests are retried on connection errors and 502/503/504. + """ + + def __init__( + self, + base_url: str, + *, + verify: bool | str = True, + timeout_s: float = 60.0, + retries: int = 2, + backoff_s: float = 0.5, + transport: BaseAdapter | None = None, + sleep: Callable[[float], None] = time.sleep, + ): + self.base_url = base_url.rstrip("/") + self.timeout_s = timeout_s + self.retries = retries + self.backoff_s = backoff_s + self._sleep = sleep + self.http = requests.Session() + self.http.verify = verify + self.http.headers.update({"Accept": "application/json"}) + if transport is not None: + self.http.mount("https://", transport) + self.http.mount("http://", transport) + self._ca_file: str | None = None + + @classmethod + def for_config(cls, config: ArrayConfig, *, transport: BaseAdapter | None = None, + base_url: str | None = None, **kw: Any) -> RestSession: + verify: bool | str = config.verify_tls + ca_file = None + if config.verify_tls and config.ca_bundle: + fd, ca_file = tempfile.mkstemp(prefix="sb-array-ca-", suffix=".pem") + with os.fdopen(fd, "w") as f: + f.write(config.ca_bundle) + verify = ca_file + session = cls(base_url or config.endpoint, verify=verify, timeout_s=config.timeout_s, + transport=transport, **kw) + session._ca_file = ca_file + return session + + # -- auth helpers --------------------------------------------------------- + + @property + def headers(self) -> Any: + return self.http.headers + + def set_basic_auth(self, username: str, password: str) -> None: + self.http.auth = (username, password) + + def clear_auth(self) -> None: + self.http.auth = None + + # -- requests ------------------------------------------------------------- + + def url(self, path: str) -> str: + if path.startswith(("http://", "https://")): + return path + return f"{self.base_url}/{path.lstrip('/')}" + + def request( + self, + method: str, + path: str, + *, + params: Any = None, + json: Any = None, + data: Any = None, + headers: dict[str, str] | None = None, + ok: Iterable[int] = (), + auth: Any = None, + ) -> requests.Response: + method = method.upper() + attempts = 1 + (self.retries if method in _IDEMPOTENT else 0) + ok_extra = frozenset(ok) + last_exc: Exception | None = None + for attempt in range(attempts): + try: + kwargs: dict[str, Any] = dict(params=params, json=json, data=data, + headers=headers, timeout=self.timeout_s) + if auth is not None: + kwargs["auth"] = auth + resp = self.http.request(method, self.url(path), **kwargs) + except requests.RequestException as e: + last_exc = e + if attempt + 1 < attempts: + self._sleep(self.backoff_s * (attempt + 1)) + continue + raise ArrayRequestError( + f"{method} {path}: {type(e).__name__}: {_safe(str(e))}") from e + if resp.status_code in _RETRY_STATUSES and attempt + 1 < attempts: + self._sleep(self.backoff_s * (attempt + 1)) + continue + if 200 <= resp.status_code < 300 or resp.status_code in ok_extra: + return resp + raise error_for(method, path, resp) + raise ArrayRequestError(f"{method} {path}: giving up") from last_exc # pragma: no cover + + def get(self, path: str, **kw: Any) -> requests.Response: + return self.request("GET", path, **kw) + + def post(self, path: str, **kw: Any) -> requests.Response: + return self.request("POST", path, **kw) + + def put(self, path: str, **kw: Any) -> requests.Response: + return self.request("PUT", path, **kw) + + def patch(self, path: str, **kw: Any) -> requests.Response: + return self.request("PATCH", path, **kw) + + def delete(self, path: str, **kw: Any) -> requests.Response: + return self.request("DELETE", path, **kw) + + def close(self) -> None: + try: + self.http.close() + finally: + if self._ca_file: + try: + os.unlink(self._ca_file) + except OSError: + pass + self._ca_file = None + + +def _safe(text: str) -> str: + return text[:_MAX_ERROR_BODY] + + +def body_text(resp: requests.Response) -> str: + try: + return _safe(resp.text or "") + except Exception: # pragma: no cover - decoding issues + return "" + + +def error_for(method: str, path: str, resp: requests.Response) -> Exception: + msg = f"{method} {path} -> HTTP {resp.status_code}: {body_text(resp)}" + if resp.status_code in (401, 403): + return ArrayAuthError(msg) + if resp.status_code == 404: + return ArrayNotFoundError(msg) + if resp.status_code == 409: + return ArrayConflictError(msg) + return ArrayRequestError(msg, status=resp.status_code) + + +def json_of(resp: requests.Response) -> Any: + """Response body as JSON (``None`` for an empty body).""" + if not resp.content: + return None + try: + return resp.json() + except ValueError as e: + raise ArrayRequestError( + f"Expected JSON from {resp.request.method} {resp.url}, got: {body_text(resp)}", + status=resp.status_code) from e diff --git a/simplyblock_core/storage_arrays/registry.py b/simplyblock_core/storage_arrays/registry.py new file mode 100644 index 0000000000..b868b75571 --- /dev/null +++ b/simplyblock_core/storage_arrays/registry.py @@ -0,0 +1,61 @@ +"""Vendor key -> adapter class. Adapters are imported lazily, on first use.""" +from __future__ import annotations + +import importlib + +from .base import ArrayAdapter, ArrayUnsupportedError + +#: vendor key -> "module:Class" (module relative to this package) +ADAPTERS: dict[str, str] = { + "pure-flasharray": "adapters.pure:PureFlashArrayAdapter", + "netapp-ontap": "adapters.netapp_ontap:NetAppOntapAdapter", + "dell-powerstore": "adapters.dell_powerstore:DellPowerStoreAdapter", + "dell-powermax": "adapters.dell_powermax:DellPowerMaxAdapter", + "dell-powerflex": "adapters.dell_powerflex:DellPowerFlexAdapter", + "hpe-wsapi": "adapters.hpe_wsapi:HpeWsapiAdapter", + "hpe-nimble": "adapters.hpe_nimble:HpeNimbleAdapter", + "ibm-flashsystem": "adapters.ibm_flashsystem:IbmFlashSystemAdapter", + "hitachi-vsp": "adapters.hitachi_vsp:HitachiVspAdapter", + "huawei-dorado": "adapters.huawei_dorado:HuaweiDoradoAdapter", + "infinidat-infinibox": "adapters.infinidat:InfinidatAdapter", +} + +#: Other names customers use for the same API. +ALIASES: dict[str, str] = { + "pure": "pure-flasharray", + "netapp": "netapp-ontap", + "netapp-asa": "netapp-ontap", + "lenovo-thinksystem-dm": "netapp-ontap", + "powerstore": "dell-powerstore", + "powermax": "dell-powermax", + "powerflex": "dell-powerflex", + "hpe-alletra-mp": "hpe-wsapi", + "hpe-primera": "hpe-wsapi", + "hpe-3par": "hpe-wsapi", + "hpe-alletra-6000": "hpe-nimble", + "hpe-alletra-5000": "hpe-nimble", + "ibm-storage-virtualize": "ibm-flashsystem", + "ibm-svc": "ibm-flashsystem", + "hitachi": "hitachi-vsp", + "huawei-oceanstor": "huawei-dorado", + "infinidat": "infinidat-infinibox", +} + + +def canonical_vendor(vendor: str) -> str: + key = vendor.strip().lower() + key = ALIASES.get(key, key) + if key not in ADAPTERS: + raise ArrayUnsupportedError( + f"Unknown array vendor {vendor!r}; supported: {', '.join(sorted(ADAPTERS))}") + return key + + +def adapter_class(vendor: str) -> type[ArrayAdapter]: + target = ADAPTERS[canonical_vendor(vendor)] + module_name, _, class_name = target.partition(":") + module = importlib.import_module(f"{__package__}.{module_name}") + cls = getattr(module, class_name) + if not (isinstance(cls, type) and issubclass(cls, ArrayAdapter)): + raise TypeError(f"{target} is not an ArrayAdapter") + return cls diff --git a/simplyblock_core/storage_arrays/testing/__init__.py b/simplyblock_core/storage_arrays/testing/__init__.py new file mode 100644 index 0000000000..b295365650 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/__init__.py @@ -0,0 +1,314 @@ +"""Stateful fakes of each vendor's management REST API. + +Each ``testing/.py`` module provides + +* a :class:`FakeArray` subclass that keeps volumes/hosts/mappings in memory + and answers the subset of the vendor API its adapter uses, with the + vendor's status codes, error bodies and quirks (async jobs, session + tokens, size units); and +* ``contract_setup()`` returning a :class:`ContractSetup` for the shared + contract suite. + +Fakes plug into ``requests`` in-process through :class:`FakeTransport` (no +sockets), or run as a real HTTPS server with :func:`serve` / ``python -m +simplyblock_core.storage_arrays.testing`` for end-to-end and UI testing. + +A fake encodes our reading of the vendor API. Passing the contract suite +proves the adapter logic; it does not prove API fidelity -- that needs the +same suite against a vendor simulator or a real array (``live`` tier). +""" +from __future__ import annotations + +import base64 +import io +import json +import re +import threading +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import parse_qs, urlsplit + +import requests +from requests.adapters import BaseAdapter +from requests.structures import CaseInsensitiveDict + +from simplyblock_core.credentials import Credential + +from ..base import ArrayConfig, Initiator, Protocol + +Handler = Callable[..., "Reply"] + + +@dataclass +class Request: + method: str + path: str + query: dict[str, list[str]] + headers: CaseInsensitiveDict + body: bytes + + def json(self) -> Any: + return json.loads(self.body) if self.body else None + + def q(self, name: str, default: str | None = None) -> str | None: + values = self.query.get(name) + return values[0] if values else default + + def basic_auth(self) -> tuple[str, str] | None: + header = self.headers.get("Authorization", "") + if not header.lower().startswith("basic "): + return None + user, _, pw = base64.b64decode(header[6:]).decode().partition(":") + return user, pw + + +@dataclass +class Reply: + status: int = 200 + body: Any = None + headers: dict[str, str] = field(default_factory=dict) + + +def route(method: str, pattern: str): + """Decorate a FakeArray method as the handler of ``method`` + path regex. + Named groups are passed as keyword arguments.""" + def deco(fn): + fn._route = (method.upper(), re.compile(f"^{pattern}$")) + return fn + return deco + + +class FakeArray: + """Base class: routing, fault injection, request log, thread safety.""" + + vendor: str = "" + + def __init__(self) -> None: + self.lock = threading.RLock() + self.requests: list[tuple[str, str]] = [] + self._faults: list[tuple[re.Pattern[str] | None, Reply, int]] = [] + self._routes: list[tuple[str, re.Pattern[str], Handler]] = [] + for name in dir(type(self)): + fn = getattr(type(self), name) + r = getattr(fn, "_route", None) + if r: + self._routes.append((r[0], r[1], getattr(self, name))) + self._id = 0 + + # -- helpers for subclasses ---------------------------------------------- + + def next_id(self) -> int: + self._id += 1 + return self._id + + # -- fault injection -------------------------------------------------------- + + def fail_next(self, status: int, body: Any = None, *, path: str | None = None, + times: int = 1) -> None: + """Answer the next ``times`` requests (optionally only those whose path + matches ``path`` regex) with ``status`` and ``body``.""" + with self.lock: + self._faults.append((re.compile(path) if path else None, Reply(status, body), times)) + + def expire_sessions(self) -> None: + """Invalidate every session token, as an array restart or timeout would. + Basic-auth fakes have nothing to expire.""" + + def set_credential(self, **fields: str) -> None: + """Change the credential the fake accepts (simulates rotation).""" + raise NotImplementedError + + # -- dispatch --------------------------------------------------------------- + + def handle(self, req: Request) -> Reply: + with self.lock: + self.requests.append((req.method, req.path)) + for i, (pattern, reply, times) in enumerate(self._faults): + if pattern is None or pattern.search(req.path): + if times <= 1: + del self._faults[i] + else: + self._faults[i] = (pattern, reply, times - 1) + return reply + for method, pattern, handler in self._routes: + if method != req.method: + continue + m = pattern.match(req.path) + if m: + try: + return handler(req, **m.groupdict()) + except FakeError as e: + return e.reply + return self.not_routed(req) + + def not_routed(self, req: Request) -> Reply: + return Reply(404, {"error": f"fake: no route for {req.method} {req.path}"}) + + +class FakeError(Exception): + """Raise inside a handler to answer with ``reply``.""" + + def __init__(self, status: int, body: Any = None, headers: dict[str, str] | None = None): + super().__init__(status) + self.reply = Reply(status, body, headers or {}) + + +class FakeTransport(BaseAdapter): + """A ``requests`` transport adapter answering from a :class:`FakeArray`.""" + + def __init__(self, fake: FakeArray): + super().__init__() + self.fake = fake + + def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): + parts = urlsplit(request.url) + body = request.body or b"" + if isinstance(body, str): + body = body.encode() + req = Request( + method=request.method.upper(), + path=parts.path, + query=parse_qs(parts.query, keep_blank_values=True), + headers=CaseInsensitiveDict(request.headers), + body=body, + ) + reply = self.fake.handle(req) + return _to_response(request, reply) + + def close(self): + pass + + +def _encode(reply: Reply) -> tuple[bytes, dict[str, str]]: + headers = dict(reply.headers) + if reply.body is None: + return b"", headers + if isinstance(reply.body, bytes): + return reply.body, headers + if isinstance(reply.body, str): + headers.setdefault("Content-Type", "text/plain") + return reply.body.encode(), headers + headers.setdefault("Content-Type", "application/json") + return json.dumps(reply.body).encode(), headers + + +def _to_response(request, reply: Reply) -> requests.Response: + content, headers = _encode(reply) + resp = requests.Response() + resp.status_code = reply.status + resp.headers = CaseInsensitiveDict(headers) + resp.raw = io.BytesIO(content) + resp._content = content + resp.url = request.url + resp.request = request + resp.encoding = "utf-8" + resp.reason = "" + return resp + + +@dataclass +class ContractSetup: + """What the shared contract suite needs from a vendor fake.""" + + fake: FakeArray + config: ArrayConfig + credential: Credential + #: Credential the fake rejects. + bad_credential: Credential + #: Two distinct initiator sets per supported protocol, for two hosts. + initiators: dict[Protocol, tuple[tuple[Initiator, ...], tuple[Initiator, ...]]] + #: Create a volume directly on the fake (bypassing the client), for the + #: "not simplyblock-owned" checks. Returns the array's volume id. + create_foreign_volume: Callable[[str, int], str] + #: Whether mapping yields a LUN/NSID number. + has_lun_numbers: bool = True + + +def make_config(vendor: str, endpoint: str, **options: Any) -> ArrayConfig: + return ArrayConfig( + id=f"fake-{vendor}", + vendor=vendor, + endpoint=endpoint, + credential_ref={"kind": "vault-kv", "mount": "m", "path": f"arrays/{vendor}"}, + options=options, + ) + + +# -- standalone HTTPS mock server ------------------------------------------------ + + +def serve(fake: FakeArray, host: str = "127.0.0.1", port: int = 8443, + certfile: str | None = None, keyfile: str | None = None): + """Serve ``fake`` over HTTPS (a throwaway self-signed certificate is + generated unless ``certfile``/``keyfile`` are given). Returns the server; + call ``serve_forever()`` on it.""" + import ssl + import tempfile + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def _do(self): + parts = urlsplit(self.path) + length = int(self.headers.get("Content-Length") or 0) + body = self.rfile.read(length) if length else b"" + reply = fake.handle(Request( + method=self.command, path=parts.path, + query=parse_qs(parts.query, keep_blank_values=True), + headers=CaseInsensitiveDict(dict(self.headers.items())), body=body)) + content, headers = _encode(reply) + self.send_response(reply.status) + for k, v in headers.items(): + self.send_header(k, v) + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + + do_GET = do_POST = do_PUT = do_PATCH = do_DELETE = _do + + def log_message(self, fmt, *args): # quieter default logging + pass + + server = ThreadingHTTPServer((host, port), Handler) + if certfile is None: + tmp = tempfile.mkdtemp(prefix="sb-array-mock-") + certfile, keyfile = f"{tmp}/cert.pem", f"{tmp}/key.pem" + _self_signed(host, certfile, keyfile) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(certfile, keyfile) + server.socket = ctx.wrap_socket(server.socket, server_side=True) + server.certfile = certfile # type: ignore[attr-defined] + return server + + +def _self_signed(host: str, certfile: str, keyfile: str) -> None: + import datetime + import ipaddress + + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + from cryptography.x509.oid import NameOID + + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, host)]) + try: + san: x509.GeneralName = x509.IPAddress(ipaddress.ip_address(host)) + except ValueError: + san = x509.DNSName(host) + now = datetime.datetime.now(datetime.UTC) + cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(minutes=5)) + .not_valid_after(now + datetime.timedelta(days=30)) + .add_extension(x509.SubjectAlternativeName([san]), critical=False) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .sign(key, hashes.SHA256())) + with open(certfile, "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + with open(keyfile, "wb") as f: + f.write(key.private_bytes(serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption())) diff --git a/simplyblock_core/storage_arrays/testing/__main__.py b/simplyblock_core/storage_arrays/testing/__main__.py new file mode 100644 index 0000000000..6eee1688aa --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/__main__.py @@ -0,0 +1,48 @@ +"""Run a vendor fake as an HTTPS mock server. + + python -m simplyblock_core.storage_arrays.testing --vendor pure-flasharray --port 8443 + +Prints the accepted credential, the CA certificate path (pass its content as +the array's ``ca_bundle``) and the options to register the array with. +""" +import argparse +import importlib +import json + +from ..registry import ADAPTERS, canonical_vendor + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--vendor", required=True, help=", ".join(sorted(ADAPTERS))) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8443) + args = parser.parse_args() + + vendor = canonical_vendor(args.vendor) + module = importlib.import_module( + f"{__package__}.{ADAPTERS[vendor].split(':')[0].split('.')[-1]}") + setup = module.contract_setup() + server = serve_fake(setup.fake, args.host, args.port) + print(json.dumps({ + "vendor": vendor, + "endpoint": f"https://{args.host}:{args.port}", + "ca_bundle_file": server.certfile, + "credential_fields": sorted(setup.credential.field_names), + "credential": {k: setup.credential.get(k).get_secret_value() + for k in setup.credential.field_names}, + "options": setup.config.options, + }, indent=2)) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + + +def serve_fake(fake, host, port): + from . import serve + return serve(fake, host, port) + + +if __name__ == "__main__": + main() diff --git a/simplyblock_core/storage_arrays/testing/pure.py b/simplyblock_core/storage_arrays/testing/pure.py new file mode 100644 index 0000000000..dd611d6bb0 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/pure.py @@ -0,0 +1,270 @@ +"""Fake Pure FlashArray REST 2.x (the subset the adapter uses).""" +from __future__ import annotations + +import secrets +from typing import Any, ClassVar + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_V = r"/api/(?P2\.\d+)" + + +def _err(msg: str, status: int = 400) -> FakeError: + return FakeError(status, {"errors": [{"message": msg, "context": ""}]}) + + +class FakePureFlashArray(FakeArray): + vendor = "pure-flasharray" + versions: ClassVar[list[str]] = ["1.19", "2.0", "2.4", "2.17", "2.21"] + + def __init__(self, api_token: str = "pure-token"): + super().__init__() + self.api_token = api_token + self.sessions: set[str] = set() + self.volumes: dict[str, dict[str, Any]] = {} # id -> volume + self.hosts: dict[str, dict[str, Any]] = {} # name -> host + self.connections: list[dict[str, Any]] = [] # {host, volume_id, lun} + + # -- fake controls ------------------------------------------------------ + + def expire_sessions(self) -> None: + self.sessions.clear() + + def set_credential(self, **fields: str) -> None: + self.api_token = fields["api_token"] + + def add_volume(self, name: str, size: int) -> str: + vid = f"{secrets.token_hex(4)}-{self.next_id()}" + self.volumes[vid] = {"id": vid, "name": name, "provisioned": size, + "serial": secrets.token_hex(12).upper(), "destroyed": False} + return vid + + # -- auth --------------------------------------------------------------- + + def _auth(self, req: Request) -> None: + if req.headers.get("x-auth-token") not in self.sessions: + raise _err("Authentication failed.", 401) + + @route("GET", "/api/api_version") + def api_version(self, req): + return Reply(200, {"version": self.versions}) + + @route("POST", _V + "/login") + def login(self, req, ver): + if req.headers.get("api-token") != self.api_token: + raise _err("Invalid API token.", 401) + token = secrets.token_hex(8) + self.sessions.add(token) + return Reply(200, {"items": [{"username": "sb"}]}, {"x-auth-token": token}) + + @route("POST", _V + "/logout") + def logout(self, req, ver): + self.sessions.discard(req.headers.get("x-auth-token")) + return Reply(200, {}) + + @route("GET", _V + "/arrays") + def arrays(self, req, ver): + self._auth(req) + return Reply(200, {"items": [{"id": "fake-array-id", "name": "fake-fa", + "os": "Purity//FA", "version": "6.5.0"}]}) + + @route("GET", _V + "/controllers") + def controllers(self, req, ver): + self._auth(req) + return Reply(200, {"items": [{"name": "CT0", "model": "FA-X70R4"}]}) + + # -- volumes ------------------------------------------------------------- + + def _select(self, req) -> list[dict[str, Any]]: + if req.q("ids"): + ids = req.q("ids").split(",") + missing = [i for i in ids if i not in self.volumes] + if missing: + raise _err("Volume does not exist.") + return [self.volumes[i] for i in ids] + names = (req.q("names") or "").split(",") + found = [v for v in self.volumes.values() if v["name"] in names] + if len(found) < len(names): + raise _err("Volume does not exist.") + return found + + @route("GET", _V + "/volumes") + def get_volumes(self, req, ver): + self._auth(req) + return Reply(200, {"items": self._select(req)}) + + @route("POST", _V + "/volumes") + def post_volumes(self, req, ver): + self._auth(req) + name = req.q("names") + body = req.json() + if any(v["name"] == name for v in self.volumes.values()): + raise _err("Volume already exists.") + size = int(body["provisioned"]) + if size % 512: + raise _err("Volume size must be a multiple of 512.") + vid = self.add_volume(name, size) + return Reply(200, {"items": [self.volumes[vid]]}) + + @route("PATCH", _V + "/volumes") + def patch_volumes(self, req, ver): + self._auth(req) + body = req.json() + vols = self._select(req) + for v in vols: + if "provisioned" in body: + size = int(body["provisioned"]) + if size < v["provisioned"] and not body.get("truncate"): + raise _err("Implicit truncation is not permitted.") + v["provisioned"] = size + if body.get("destroyed"): + if any(c["volume_id"] == v["id"] for c in self.connections): + raise _err("Volume has host connections.") + v["destroyed"] = True + return Reply(200, {"items": vols}) + + @route("DELETE", _V + "/volumes") + def delete_volumes(self, req, ver): + self._auth(req) + for v in self._select(req): + if not v["destroyed"]: + raise _err("Volume must be destroyed before eradication.") + del self.volumes[v["id"]] + return Reply(200, {}) + + # -- hosts ---------------------------------------------------------------- + + def _owner(self, key: str, value: str) -> str | None: + for h in self.hosts.values(): + if value.lower() in (x.lower() for x in h[key]): + return h["name"] + return None + + @route("GET", _V + "/hosts") + def get_hosts(self, req, ver): + self._auth(req) + name = req.q("names") + if name not in self.hosts: + raise _err("Host does not exist.") + return Reply(200, {"items": [self.hosts[name]]}) + + def _claim(self, host: str, body: dict[str, Any], prefix: str = "") -> None: + for key in ("iqns", "wwns", "nqns"): + for value in body.get(prefix + key, []): + owner = self._owner(key, value) + if owner and owner != host: + raise _err(f"{value} is already in use by host {owner}.") + if key == "wwns" and len(value.replace(":", "")) != 16: + raise _err("Invalid WWN.") + if value.lower() not in (x.lower() for x in self.hosts[host][key]): + self.hosts[host][key].append(value) + + @route("POST", _V + "/hosts") + def post_hosts(self, req, ver): + self._auth(req) + name = req.q("names") + if name in self.hosts: + raise _err("Host already exists.") + self.hosts[name] = {"name": name, "iqns": [], "wwns": [], "nqns": []} + try: + self._claim(name, req.json() or {}) + except FakeError: + del self.hosts[name] + raise + return Reply(200, {"items": [self.hosts[name]]}) + + @route("PATCH", _V + "/hosts") + def patch_hosts(self, req, ver): + self._auth(req) + name = req.q("names") + if name not in self.hosts: + raise _err("Host does not exist.") + self._claim(name, req.json() or {}, prefix="add_") + return Reply(200, {"items": [self.hosts[name]]}) + + @route("DELETE", _V + "/hosts") + def delete_hosts(self, req, ver): + self._auth(req) + name = req.q("names") + if name not in self.hosts: + raise _err("Host does not exist.") + if any(c["host"] == name for c in self.connections): + raise _err("Host has volume connections.") + del self.hosts[name] + return Reply(200, {}) + + # -- connections ------------------------------------------------------------ + + def _conn_item(self, c): + v = self.volumes[c["volume_id"]] + return {"host": {"name": c["host"]}, "volume": {"id": v["id"], "name": v["name"]}, + "lun": c["lun"]} + + @route("GET", _V + "/connections") + def get_connections(self, req, ver): + self._auth(req) + vid, host = req.q("volume_ids"), req.q("host_names") + if vid and vid not in self.volumes: + raise _err("Volume does not exist.") + items = [self._conn_item(c) for c in self.connections + if (not vid or c["volume_id"] == vid) and (not host or c["host"] == host)] + return Reply(200, {"items": items}) + + @route("POST", _V + "/connections") + def post_connections(self, req, ver): + self._auth(req) + vid, host = req.q("volume_ids"), req.q("host_names") + if vid not in self.volumes or self.volumes[vid]["destroyed"]: + raise _err("Volume does not exist.") + if host not in self.hosts: + raise _err("Host does not exist.") + if any(c["volume_id"] == vid and c["host"] == host for c in self.connections): + raise _err("Connection already exists.") + used = {c["lun"] for c in self.connections if c["host"] == host} + lun = next(n for n in range(1, 4096) if n not in used) + c = {"host": host, "volume_id": vid, "lun": lun} + self.connections.append(c) + return Reply(200, {"items": [self._conn_item(c)]}) + + @route("DELETE", _V + "/connections") + def delete_connections(self, req, ver): + self._auth(req) + vid, host = req.q("volume_ids"), req.q("host_names") + before = len(self.connections) + self.connections = [c for c in self.connections + if not (c["volume_id"] == vid and c["host"] == host)] + if len(self.connections) == before: + raise _err("Connection does not exist.") + return Reply(200, {}) + + +def contract_setup() -> ContractSetup: + fake = FakePureFlashArray(api_token="pure-token") + return ContractSetup( + fake=fake, + config=make_config("pure-flasharray", "https://pure.fake.example"), + credential=Credential({"api_token": SecretStr("pure-token")}), + bad_credential=Credential({"api_token": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + Protocol.NVME_TCP: ( + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-a"),), + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-b"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/tests/unit/storage_arrays/__init__.py b/tests/unit/storage_arrays/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/storage_arrays/conftest.py b/tests/unit/storage_arrays/conftest.py new file mode 100644 index 0000000000..44d8985189 --- /dev/null +++ b/tests/unit/storage_arrays/conftest.py @@ -0,0 +1,115 @@ +"""Shared fixtures: run ArrayClient against a vendor fake (in-process) or, +with ``SB_ARRAY_LIVE_CONFIG=``, against a real array/simulator. + +Live config file:: + + {"vendor": "netapp-ontap", "endpoint": "https://cluster1.lab:443", + "credential": {"username": "...", "password": "..."}, + "ca_bundle": "-----BEGIN CERTIFICATE-----...", "verify_tls": true, + "options": {"svm": "svm1"}, + "initiators": {"iscsi": [["iqn...a"], ["iqn...b"]]}} + +The live tier creates and deletes only ``sb-`` volumes and ``sbtest-`` hosts. +""" +from __future__ import annotations + +import importlib +import json +import os + +import pytest +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential +from simplyblock_core.storage_arrays.base import ArrayConfig, Initiator, Protocol +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.registry import ADAPTERS +from simplyblock_core.storage_arrays.testing import ContractSetup, FakeTransport + +VENDORS = sorted(ADAPTERS) + + +class StaticProvider: + """Stands in for CredentialProvider: returns ``credential`` and counts + invalidations. Tests flip ``credential`` to simulate rotation.""" + + def __init__(self, credential: Credential): + self.credential = credential + self.resolved: list[str] = [] + self.invalidated = 0 + + def resolve(self, ref, context): + self.resolved.append(context) + return self.credential + + def invalidate(self, ref=None): + self.invalidated += 1 + + +def fake_module(vendor: str): + tail = ADAPTERS[vendor].split(":")[0].split(".")[-1] + try: + return importlib.import_module(f"simplyblock_core.storage_arrays.testing.{tail}") + except ModuleNotFoundError as e: + if e.name and e.name.endswith(tail): + pytest.skip(f"no fake for {vendor} yet") + raise + + +class Harness: + def __init__(self, setup: ContractSetup, live: bool = False): + self.setup = setup + self.live = live + self.provider = StaticProvider(setup.credential) + self.client = ArrayClient( + setup.config, self.provider, # type: ignore[arg-type] + transport=None if live else FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}, + ) + + @property + def fake(self): + return self.setup.fake + + +def _live_setup(path: str) -> ContractSetup: + with open(path) as f: + cfg = json.load(f) + initiators = { + Protocol(proto): tuple( + tuple(Initiator(protocol=Protocol(proto), address=a) for a in group) + for group in groups) + for proto, groups in cfg["initiators"].items() + } + return ContractSetup( + fake=None, # type: ignore[arg-type] + config=ArrayConfig(id="live", vendor=cfg["vendor"], endpoint=cfg["endpoint"], + credential_ref={"kind": "vault-kv", "mount": "m", "path": "p"}, + ca_bundle=cfg.get("ca_bundle"), + verify_tls=cfg.get("verify_tls", True), + options=cfg.get("options", {})), + credential=Credential({k: SecretStr(v) for k, v in cfg["credential"].items()}), + bad_credential=Credential({k: SecretStr("wrong-" + v) + for k, v in cfg["credential"].items()}), + initiators=initiators, + create_foreign_volume=None, # type: ignore[arg-type] + has_lun_numbers=cfg.get("has_lun_numbers", True), + ) + + +def _params(): + live = os.environ.get("SB_ARRAY_LIVE_CONFIG") + if live: + return [pytest.param(("live", live), id="live")] + return [pytest.param(("fake", v), id=v) for v in VENDORS] + + +@pytest.fixture(params=_params()) +def harness(request) -> Harness: + kind, value = request.param + if kind == "live": + h = Harness(_live_setup(value), live=True) + else: + h = Harness(fake_module(value).contract_setup()) + yield h + h.client.close() diff --git a/tests/unit/storage_arrays/test_contract.py b/tests/unit/storage_arrays/test_contract.py new file mode 100644 index 0000000000..6e36507a59 --- /dev/null +++ b/tests/unit/storage_arrays/test_contract.py @@ -0,0 +1,270 @@ +"""The contract every array adapter must meet, run against each vendor fake +(and, with SB_ARRAY_LIVE_CONFIG, against one real array or simulator).""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayRequestError, + HostSpec, + Protocol, +) +from simplyblock_core.storage_arrays.client import MANAGED_PREFIX, round_size + +GIB = 1 << 30 + + +def _uuid() -> str: + return str(uuid.uuid4()) + + +def _protocols(h): + return [p for p in h.setup.initiators if p in h.client.capabilities.protocols] + + +def _host(h, protocol, which=0, name=None): + return HostSpec(name=name or f"sbtest-{protocol.value}-{which}-{uuid.uuid4().hex[:6]}", + initiators=h.setup.initiators[protocol][which]) + + +@pytest.fixture() +def cleanup(harness): + volumes, hosts = [], [] + yield volumes, hosts + for vid in volumes: + try: + harness.client.delete_volume(vid) + except ArrayError: + pass + for host in hosts: + try: + harness.client.delete_host(host) + except ArrayError: + pass + + +def test_adapter_declares_itself(harness): + cls = harness.client.adapter_cls + assert cls.vendor == harness.client.vendor + assert cls.display_name + caps = cls.capabilities + assert caps.protocols + assert caps.size_granularity_bytes > 0 and caps.min_size_bytes > 0 + assert caps.max_name_len >= len(MANAGED_PREFIX) + 16 + assert set(_protocols(harness)), "fake provides no initiators for a supported protocol" + + +def test_get_info(harness): + info = harness.client.get_info() + assert info.vendor == harness.client.vendor + assert harness.provider.resolved[0].startswith("array/") + + +def test_create_get_find(harness, cleanup): + vol_uuid = _uuid() + vol = harness.client.create_volume(vol_uuid, 10 * GIB + 1) + cleanup[0].append(vol.id) + caps = harness.client.capabilities + assert vol.name == harness.client.volume_name(vol_uuid) + assert vol.name.startswith(MANAGED_PREFIX) + assert len(vol.name) <= caps.max_name_len + assert vol.size_bytes == round_size(10 * GIB + 1, caps.size_granularity_bytes, + caps.min_size_bytes) + assert vol.identifier and vol.identifier == vol.identifier.lower() + assert vol.wwn is None or vol.wwn.startswith("naa.") + assert vol.nguid is None or vol.nguid.startswith("eui.") + + got = harness.client.get_volume(vol.id) + assert got == vol + assert harness.client.find_volume(vol_uuid) == vol + + +def test_missing_volume_is_none(harness): + assert harness.client.find_volume(_uuid()) is None + + +def test_get_volume_of_deleted_is_none(harness, cleanup): + vol = harness.client.create_volume(_uuid(), GIB) + assert harness.client.delete_volume(vol.id) is True + assert harness.client.get_volume(vol.id) is None + assert harness.client.delete_volume(vol.id) is False + + +def test_create_is_idempotent(harness, cleanup): + vol_uuid = _uuid() + first = harness.client.create_volume(vol_uuid, 4 * GIB) + cleanup[0].append(first.id) + again = harness.client.create_volume(vol_uuid, 4 * GIB) + assert again.id == first.id and again.identifier == first.identifier + + +def test_create_rerun_with_larger_size_grows(harness, cleanup): + vol_uuid = _uuid() + first = harness.client.create_volume(vol_uuid, 8 * GIB) + cleanup[0].append(first.id) + again = harness.client.create_volume(vol_uuid, 16 * GIB) + assert again.id == first.id and again.size_bytes >= 16 * GIB + + +def test_resize(harness, cleanup): + vol = harness.client.create_volume(_uuid(), 8 * GIB) + cleanup[0].append(vol.id) + grown = harness.client.resize_volume(vol.id, 24 * GIB, expected_identifier=vol.identifier) + assert grown.size_bytes >= 24 * GIB + assert grown.identifier == vol.identifier + assert harness.client.get_volume(vol.id).size_bytes == grown.size_bytes + # same size: no-op + assert harness.client.resize_volume(vol.id, grown.size_bytes) == grown + with pytest.raises(ArrayConflictError, match="shrink"): + harness.client.resize_volume(vol.id, 8 * GIB) + + +def test_host_and_mapping_lifecycle(harness, cleanup): + for protocol in _protocols(harness): + vol = harness.client.create_volume(_uuid(), GIB * 8) + cleanup[0].append(vol.id) + spec_a = _host(harness, protocol, 0) + spec_b = _host(harness, protocol, 1) + host_a = harness.client.ensure_host(spec_a) + cleanup[1].append(host_a) + assert harness.client.ensure_host(spec_a) == host_a, "ensure_host not idempotent" + host_b = harness.client.ensure_host(spec_b) + cleanup[1].append(host_b) + assert host_a != host_b + + m_a = harness.client.map_volume(vol.id, host_a, expected_identifier=vol.identifier) + assert m_a.host_id == host_a and m_a.volume_id == vol.id + if harness.setup.has_lun_numbers: + assert isinstance(m_a.lun, int) and m_a.lun >= 0 + assert harness.client.map_volume(vol.id, host_a) == m_a, "map not idempotent" + m_b = harness.client.map_volume(vol.id, host_b) # multi-attach + + hosts = {m.host_id for m in harness.client.list_mappings(vol.id)} + assert hosts == {host_a, host_b} + + harness.client.unmap_volume(vol.id, host_a) + harness.client.unmap_volume(vol.id, host_a) # idempotent + assert [m.host_id for m in harness.client.list_mappings(vol.id)] == [m_b.host_id] + + # delete unmaps the remaining host first + assert harness.client.delete_volume(vol.id, expected_identifier=vol.identifier) + cleanup[0].remove(vol.id) + assert harness.client.get_volume(vol.id) is None + + harness.client.delete_host(host_a) + harness.client.delete_host(host_a) # absent: no-op + harness.client.delete_host(host_b) + cleanup[1].clear() + + +def test_initiator_owned_by_another_host_conflicts(harness, cleanup): + protocol = _protocols(harness)[0] + first = _host(harness, protocol, 0) + cleanup[1].append(harness.client.ensure_host(first)) + other = HostSpec(name=f"sbtest-other-{uuid.uuid4().hex[:6]}", initiators=first.initiators) + with pytest.raises(ArrayConflictError): + cleanup[1].append(harness.client.ensure_host(other)) + + +def test_existing_host_gets_missing_initiators(harness, cleanup): + protocol = _protocols(harness)[0] + a, b = harness.setup.initiators[protocol] + name = f"sbtest-grow-{uuid.uuid4().hex[:6]}" + host = harness.client.ensure_host(HostSpec(name=name, initiators=a)) + cleanup[1].append(host) + assert harness.client.ensure_host(HostSpec(name=name, initiators=a + b)) == host + # b's initiators now belong to that host: claiming them elsewhere conflicts + with pytest.raises(ArrayConflictError): + cleanup[1].append(harness.client.ensure_host( + HostSpec(name=f"sbtest-x-{uuid.uuid4().hex[:6]}", initiators=b))) + + +def test_wrong_identifier_is_refused(harness, cleanup): + vol = harness.client.create_volume(_uuid(), GIB) + cleanup[0].append(vol.id) + for action in ( + lambda: harness.client.delete_volume(vol.id, expected_identifier="naa.600000000000"), + lambda: harness.client.resize_volume(vol.id, 4 * GIB, expected_identifier="naa.6000"), + ): + with pytest.raises(ArrayConflictError, match="does not match"): + action() + assert harness.client.get_volume(vol.id) is not None + + +def test_foreign_volume_is_refused(harness): + if harness.live: + pytest.skip("live tier never touches foreign volumes") + vid = harness.setup.create_foreign_volume("customer-db01", 4 * GIB) + for action in (lambda: harness.client.delete_volume(vid), + lambda: harness.client.resize_volume(vid, 8 * GIB)): + with pytest.raises(ArrayConflictError, match="not created by simplyblock"): + action() + assert harness.client.get_volume(vid) is not None + + +def test_session_expiry_is_transparent(harness, cleanup): + if harness.live: + pytest.skip("cannot expire a real array's sessions") + harness.client.get_info() + harness.fake.expire_sessions() + vol = harness.client.create_volume(_uuid(), GIB) + cleanup[0].append(vol.id) + assert harness.client.get_volume(vol.id) == vol + + +def test_bad_credential_raises_auth_error_after_one_retry(harness): + harness.provider.credential = harness.setup.bad_credential + with pytest.raises(ArrayAuthError): + harness.client.get_info() + assert harness.provider.invalidated == 1 + assert len(harness.provider.resolved) == 2 + + +def test_rotated_credential_is_picked_up(harness): + if harness.live: + pytest.skip("needs control of the array's credential") + harness.client.get_info() + new = {k: "rotated-" + harness.setup.credential.get(k).get_secret_value() + for k in harness.setup.credential.field_names} + harness.fake.set_credential(**new) + harness.fake.expire_sessions() + from pydantic import SecretStr + + from simplyblock_core.credentials import Credential + old_secret_values = {harness.setup.credential.get(k).get_secret_value() + for k in harness.setup.credential.field_names} + harness.provider.credential = Credential({k: SecretStr(v) for k, v in new.items()}) + # A basic-auth adapter keeps the old password in its session until it is + # rejected; either way the client must recover with one invalidation. + harness.client.get_info() + assert harness.provider.invalidated <= 1 + assert old_secret_values # silence unused + + +def test_server_error_does_not_leak_credentials(harness): + if harness.live: + pytest.skip("fault injection needs the fake") + harness.client.get_info() + harness.fake.fail_next(500, {"error": "internal"}, times=50) + with pytest.raises(ArrayRequestError) as e: + harness.client.create_volume(_uuid(), GIB) + text = str(e.value) + for name in harness.setup.credential.field_names: + assert harness.setup.credential.get(name).get_secret_value() not in text + + +def test_unsupported_protocol_is_refused_before_calling_the_array(harness): + unsupported = [p for p in Protocol if p not in harness.client.capabilities.protocols] + if not unsupported: + pytest.skip("adapter supports every protocol") + from simplyblock_core.storage_arrays.base import Initiator + p = unsupported[0] + address = "21:00:00:24:ff:4c:1a:99" if p in (Protocol.FC, Protocol.NVME_FC) else "x" + with pytest.raises(ArrayError, match="does not support"): + harness.client.ensure_host(HostSpec(name="sbtest-unsup", + initiators=(Initiator(protocol=p, address=address),))) From e90d205ddd41f3718eaa66dd1df9b052274dc71e Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 14:15:06 +0100 Subject: [PATCH 29/50] feat(storage_arrays): adapters for all supported array families Ten more adapters behind the ArrayAdapter contract, each with a stateful fake of the vendor REST API and vendor-specific tests: NetApp ONTAP, Dell PowerStore / PowerMax / PowerFlex, HPE WSAPI (Alletra MP, Primera, 3PAR), HPE Nimble (Alletra 5000/6000), IBM FlashSystem, Hitachi VSP, Huawei OceanStor Dorado, Infinidat InfiniBox. Framework follow-ups from integrating them: - Volume.wwn holds any SCSI designator (naa. or eui. for Nimble). - NVMe/FC initiators are NQNs, not WWPNs. - Capabilities.max_initiators_per_host (PowerFlex SDC hosts = 1). - Client pins resize/map/delete to the name derived from the recorded volume UUID (array ids and WWNs can be reused after delete, e.g. VSP). - RestSession.sleep is public (async job polling without real waits). - Fake transport fills the cookie jar like a real response. - Contract: delete_host must not drop live mappings (Pure now checks). Each adapter lists unverified API details under "Verify against a real system"; nothing has run against real hardware yet. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/storage_arrays/README.md | 69 ++ .../storage_arrays/adapters/dell_powerflex.py | 351 ++++++++ .../storage_arrays/adapters/dell_powermax.py | 434 +++++++++ .../adapters/dell_powerstore.py | 332 +++++++ .../storage_arrays/adapters/hitachi_vsp.py | 570 ++++++++++++ .../storage_arrays/adapters/hpe_nimble.py | 321 +++++++ .../storage_arrays/adapters/hpe_wsapi.py | 317 +++++++ .../storage_arrays/adapters/huawei_dorado.py | 426 +++++++++ .../adapters/ibm_flashsystem.py | 378 ++++++++ .../storage_arrays/adapters/infinidat.py | 290 ++++++ .../storage_arrays/adapters/netapp_ontap.py | 799 +++++++++++++++++ .../storage_arrays/adapters/pure.py | 4 + simplyblock_core/storage_arrays/base.py | 13 +- simplyblock_core/storage_arrays/client.py | 36 +- simplyblock_core/storage_arrays/http.py | 8 +- .../storage_arrays/testing/__init__.py | 17 +- .../storage_arrays/testing/dell_powerflex.py | 289 ++++++ .../storage_arrays/testing/dell_powermax.py | 593 +++++++++++++ .../storage_arrays/testing/dell_powerstore.py | 426 +++++++++ .../storage_arrays/testing/hitachi_vsp.py | 616 +++++++++++++ .../storage_arrays/testing/hpe_nimble.py | 363 ++++++++ .../storage_arrays/testing/hpe_wsapi.py | 321 +++++++ .../storage_arrays/testing/huawei_dorado.py | 589 +++++++++++++ .../storage_arrays/testing/ibm_flashsystem.py | 395 +++++++++ .../storage_arrays/testing/infinidat.py | 315 +++++++ .../storage_arrays/testing/netapp_ontap.py | 832 ++++++++++++++++++ tests/unit/storage_arrays/test_contract.py | 46 +- .../storage_arrays/test_dell_powerflex.py | 189 ++++ .../unit/storage_arrays/test_dell_powermax.py | 256 ++++++ .../storage_arrays/test_dell_powerstore.py | 212 +++++ tests/unit/storage_arrays/test_framework.py | 223 +++++ tests/unit/storage_arrays/test_hitachi_vsp.py | 360 ++++++++ tests/unit/storage_arrays/test_hpe_nimble.py | 188 ++++ tests/unit/storage_arrays/test_hpe_wsapi.py | 188 ++++ .../unit/storage_arrays/test_huawei_dorado.py | 172 ++++ .../storage_arrays/test_ibm_flashsystem.py | 187 ++++ tests/unit/storage_arrays/test_infinidat.py | 147 ++++ .../unit/storage_arrays/test_netapp_ontap.py | 430 +++++++++ 38 files changed, 11680 insertions(+), 22 deletions(-) create mode 100644 simplyblock_core/storage_arrays/README.md create mode 100644 simplyblock_core/storage_arrays/adapters/dell_powerflex.py create mode 100644 simplyblock_core/storage_arrays/adapters/dell_powermax.py create mode 100644 simplyblock_core/storage_arrays/adapters/dell_powerstore.py create mode 100644 simplyblock_core/storage_arrays/adapters/hitachi_vsp.py create mode 100644 simplyblock_core/storage_arrays/adapters/hpe_nimble.py create mode 100644 simplyblock_core/storage_arrays/adapters/hpe_wsapi.py create mode 100644 simplyblock_core/storage_arrays/adapters/huawei_dorado.py create mode 100644 simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py create mode 100644 simplyblock_core/storage_arrays/adapters/infinidat.py create mode 100644 simplyblock_core/storage_arrays/adapters/netapp_ontap.py create mode 100644 simplyblock_core/storage_arrays/testing/dell_powerflex.py create mode 100644 simplyblock_core/storage_arrays/testing/dell_powermax.py create mode 100644 simplyblock_core/storage_arrays/testing/dell_powerstore.py create mode 100644 simplyblock_core/storage_arrays/testing/hitachi_vsp.py create mode 100644 simplyblock_core/storage_arrays/testing/hpe_nimble.py create mode 100644 simplyblock_core/storage_arrays/testing/hpe_wsapi.py create mode 100644 simplyblock_core/storage_arrays/testing/huawei_dorado.py create mode 100644 simplyblock_core/storage_arrays/testing/ibm_flashsystem.py create mode 100644 simplyblock_core/storage_arrays/testing/infinidat.py create mode 100644 simplyblock_core/storage_arrays/testing/netapp_ontap.py create mode 100644 tests/unit/storage_arrays/test_dell_powerflex.py create mode 100644 tests/unit/storage_arrays/test_dell_powermax.py create mode 100644 tests/unit/storage_arrays/test_dell_powerstore.py create mode 100644 tests/unit/storage_arrays/test_framework.py create mode 100644 tests/unit/storage_arrays/test_hitachi_vsp.py create mode 100644 tests/unit/storage_arrays/test_hpe_nimble.py create mode 100644 tests/unit/storage_arrays/test_hpe_wsapi.py create mode 100644 tests/unit/storage_arrays/test_huawei_dorado.py create mode 100644 tests/unit/storage_arrays/test_ibm_flashsystem.py create mode 100644 tests/unit/storage_arrays/test_infinidat.py create mode 100644 tests/unit/storage_arrays/test_netapp_ontap.py diff --git a/simplyblock_core/storage_arrays/README.md b/simplyblock_core/storage_arrays/README.md new file mode 100644 index 0000000000..83bbbd7696 --- /dev/null +++ b/simplyblock_core/storage_arrays/README.md @@ -0,0 +1,69 @@ +# External storage arrays + +Provision, resize and delete volumes (LUNs and NVMe namespaces) on enterprise arrays, and map them to hosts. +There is no usable cross-vendor standard, so each array has its own adapter. SMI-S is deprecated, Swordfish has almost no array support, and CSI drivers depend on Kubernetes. + +## Supported systems + +| Key | Systems | API | Protocols | +|---|---|---|---| +| `pure-flasharray` | Pure FlashArray //X, //C, //XL, //E | REST 2.x, API token | iSCSI, FC, NVMe/TCP, /RoCE, /FC | +| `netapp-ontap` | NetApp AFF/FAS/ASA (not ASA r2), Lenovo DM | ONTAP REST | iSCSI, FC, NVMe/TCP, /FC (`protocol_family`) | +| `dell-powerstore` | Dell PowerStore | REST | iSCSI, FC, NVMe/TCP, /FC | +| `dell-powermax` | Dell PowerMax | Unisphere REST | iSCSI, FC, NVMe (port-group per protocol) | +| `dell-powerflex` | Dell PowerFlex 3.6 / 4.x | Gateway REST / PFxM OAuth | SDC | +| `hpe-wsapi` | HPE Alletra Storage MP B10000, Primera, 3PAR | WSAPI | iSCSI, FC | +| `hpe-nimble` | HPE Alletra 5000/6000, Nimble | NimbleOS REST | iSCSI, FC | +| `ibm-flashsystem` | IBM FlashSystem, Storage Virtualize/SVC | REST (port 7443) | iSCSI, FC, NVMe/TCP, /RDMA | +| `hitachi-vsp` | Hitachi VSP One Block, VSP E, VSP 5000 | Configuration Manager REST | iSCSI, FC | +| `huawei-dorado` | Huawei OceanStor Dorado | DeviceManager REST | iSCSI, FC | +| `infinidat-infinibox` | Infinidat InfiniBox | REST | iSCSI, FC | + +Each adapter's module docstring lists the options it reads, the credential fields it needs, and a section called **"Verify against a real system"**. +That section names every API detail its author was unsure of. Work through it before the adapter is first used against a real array. + +## Layers + +- `base.py`: the adapter contract (`ArrayAdapter`) and the shared model. +- `client.py`: `ArrayClient`, the only entry point for the rest of simplyblock. It handles: + - credentials through `simplyblock_core.credentials`, bound to the array id and endpoint, with re-authentication and one retry on auth failure; + - `sb-` volume names and rounding sizes to the array's granularity; + - making create, map, unmap and delete safe to re-run; + - safety checks before resize, map or delete: the `sb-` name prefix, the name derived from the recorded volume UUID, and the recorded WWN/NGUID; + - refusing to shrink a volume. +- `http.py`: TLS (pinned CA bundle per array), timeouts, retries, error mapping. +- `registry.py`: maps vendor keys and aliases to adapters, loading each adapter on first use. + +## Testing + +1. **Contract suite against fakes** (`tests/unit/storage_arrays/test_contract.py`). + `testing/.py` is a stateful in-memory fake of the vendor's REST API: paths, JSON shapes, status codes, error bodies, async jobs, session tokens. + Every adapter runs the same suite against its fake, through a `requests` transport, with no network. + This shows the adapter's logic is right. It does **not** show the fake matches the real API; the fakes encode our reading of each vendor's API. +2. **Vendor unit tests** (`test_.py`): unit conversions, identifier derivation, job polling and failures, error mapping, options. +3. **HTTPS mock server**, for end-to-end and UI work: + ``` + python -m simplyblock_core.storage_arrays.testing --vendor netapp-ontap --port 8443 + ``` + It prints the endpoint, the CA file, the credential and the options to register. +4. **Live tier**: the same contract suite against a simulator or a real array: + ``` + SB_ARRAY_LIVE_CONFIG=lab-array.json python -m pytest tests/unit/storage_arrays/test_contract.py --timeout-method=thread + ``` + See `tests/unit/storage_arrays/conftest.py` for the file format. The live tier only creates and deletes `sb-` volumes and `sbtest-` hosts. + +### Where to get something real to test against + +These are partly from memory; check availability with each vendor. + +- **NetApp:** ONTAP Select evaluation, and the ONTAP simulator (support-site login). Both have the full REST API. A Lab on Demand is also an option. +- **Pure:** Pure Test Drive labs. There is no downloadable FlashArray simulator. +- **Dell:** PowerFlex can be installed on 3 VMs. Dell demo labs cover PowerStore and PowerMax. PowerStore has a virtual edition for partners. +- **HPE:** the 3PAR StoreServ simulator (older WSAPI). Alletra through HPE demo and partner labs. +- **IBM:** Storage Virtualize for Public Cloud, plus IBM Technology Zone labs. +- **Hitachi, Huawei:** partner and demo labs only. +- **Infinidat:** `infinisim`, an open-source Python simulator of the InfiniBox API. + +Vendor SDKs make good reference code for API details: `py-pure-client`, `netapp-ontap`, `PyPowerStore`, `PyU4V`, `PyPowerFlex`, `hpe3parclient`, `nimble-sdk`, `infinisdk`. +The same goes for each vendor's Ansible collection, CSI driver and OpenStack Cinder driver. +This package uses plain `requests` and does not depend on any of them. diff --git a/simplyblock_core/storage_arrays/adapters/dell_powerflex.py b/simplyblock_core/storage_arrays/adapters/dell_powerflex.py new file mode 100644 index 0000000000..12e017bb2f --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/dell_powerflex.py @@ -0,0 +1,351 @@ +"""Dell PowerFlex (formerly ScaleIO / VxFlex OS), REST gateway API, PowerFlex +3.6 and 4.x. + +Auth (option ``auth``): + +* ``token`` (3.x gateway): ``GET /api/login`` with HTTP basic + ``username:password`` returns a session token as a JSON *string*; every + later request uses basic auth ``username:``. The token expires after + 8 h, or 10 min idle (-> 401; the client logs in again). + ``GET /api/logout`` invalidates it. +* ``oauth`` (4.x, PowerFlex Manager gateway): ``POST /rest/auth/login`` + ``{"username", "password"}`` returns ``{"access_token", "refresh_token", + ...}``; requests carry ``Authorization: Bearer ``. The + ``/api/...`` paths below are unchanged behind the 4.x gateway. + ``POST /rest/auth/logout {"refresh_token"}`` ends the session. The access + token is short-lived (minutes); rather than refreshing it, the adapter + logs in again on the 401 (``ArrayClient`` does this transparently). +* ``auto`` (default): try ``oauth``; a 404/405 from ``/rest/auth/login`` + means a 3.x gateway, so fall back to ``token``. + +Volumes are addressed by their 16-hex-digit id (``Volume::``). Sizes +are in KiB on create and GiB on resize and are rounded up by the array to a +multiple of 8 GiB, so the adapter declares 8 GiB granularity and minimum. +Names are at most 31 characters. Volumes are created thin +(``ThinProvisioned``) in the ``storage_pool_id`` option's pool. + +Hosts: PowerFlex's native protocol is the SDC (storage data client) kernel +driver, which registers itself with the MDM when installed on the host. The +adapter never creates SDCs: ``ensure_host`` looks the SDC up by its GUID +(the initiator address, ``drv_cfg --query_guid`` on the host) and returns +its id as the host id. An SDC is one OS instance, so a host has exactly one +SDC GUID; specs with more are refused (:class:`ArrayUnsupportedError`). + +SDC names stand in for host ownership (option ``sdc_naming``): + +* ``claim`` (default): an unnamed SDC is named after the host + (``setSdcName``); an SDC already carrying the *same* name is that host; + an SDC carrying a *different* name belongs to another host and raises + :class:`ArrayConflictError` (an admin-named SDC must therefore be + registered under its PowerFlex name). +* ``ignore``: names are neither read nor written; any host name maps to the + SDC with the given GUID (no ownership conflict detection). + +``delete_host`` is a no-op for SDCs: the SDC belongs to the host's PowerFlex +installation (removing it is ``removeSdc``, an admin decision) and keeps the +name ``claim`` gave it. + +Mappings are SDC mappings (``addMappedSdc`` / ``removeMappedSdc``); the +volume's ``mappedSdcInfo`` lists them (``null``, not ``[]``, when there are +none). Mapping a volume to a second SDC needs ``allowMultipleMappings``. +SDC mappings have no LUN number (``Mapping.lun`` is ``None``) and SDC +volumes no NAA WWN or NGUID, so ``Volume.identifier`` is ``id.``. + +NVMe/TCP (4.x NVMe hosts, ``/api/types/Host/instances``) is not +implemented: a 4.x NVMe host has exactly one NQN, and the mapping actions +for NVMe hosts could not be verified against documentation (see below). + +Errors: the gateway answers ``{"message", "httpStatusCode", "errorCode"}``, +and reports "not found" as HTTP **500** (sometimes 400) rather than 404, so +missing objects are recognised by message text; any other 500 stays an +:class:`ArrayRequestError`. Booleans and numbers in action bodies are sent as +strings (``"TRUE"``, ``"8388608"``), as the REST reference shows them. + +Verify against a real system: + +* ``auto`` detection: that a 3.x gateway answers ``POST /rest/auth/login`` + with 404/405 (not 401), and whether a 4.x gateway still serves + ``GET /api/login``. +* 4.x token lifetime and whether a refresh-token grant would be preferable + to re-login; the 4.x logout body. +* Exact error texts/codes: volume not found (``"Could not find the + volume"``, errorCode 79?), ``queryIdByKey`` for a missing name, name in + use, "already mapped", "not mapped", SDC name in use. +* That ``setVolumeSize`` rounds a non-multiple of 8 up rather than + rejecting it (the adapter always sends multiples of 8, so this only + matters for volumes created outside simplyblock). +* ``accessMode: ReadWrite`` accepted by ``addMappedSdc`` on 3.6 and 4.x; + ``allowMultipleMappings`` spelling/value ``"TRUE"``. +* Field names ``sdcGuid`` and ``name`` on ``Sdc`` and ``mappedSdcInfo[].sdcId`` + on ``Volume``; ``systemVersionName`` on ``System``. +* Whether SDC names are unique system-wide (the fake assumes so). +""" +from __future__ import annotations + +from typing import Any + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import body_text, json_of + +GIB = 1 << 30 +KIB = 1 << 10 + +_AUTH_MODES = ("auto", "token", "oauth") +_NAMING_MODES = ("claim", "ignore") +_NOT_FOUND = ("could not find", "not found", "does not exist", "no such") +_CONFLICT = ("already in use", "already exists", "already mapped", "is mapped", + "name in use") +_NOT_MAPPED = ("not mapped",) + + +def _message(body: Any, fallback: str) -> str: + if isinstance(body, dict) and body.get("message"): + msg = str(body["message"]) + code = body.get("errorCode") + return f"{msg} (errorCode {code})" if code is not None else msg + return fallback + + +class DellPowerFlexAdapter(ArrayAdapter): + vendor = "dell-powerflex" + display_name = "Dell PowerFlex" + capabilities = Capabilities( + protocols=frozenset({Protocol.SDC}), + size_granularity_bytes=8 * GIB, + min_size_bytes=8 * GIB, + max_name_len=31, + max_initiators_per_host=1, + ) + option_docs = ( + OptionDoc("storage_pool_id", "Id of the storage pool new volumes are created in.", + required=True), + OptionDoc("protection_domain_id", "Protection domain of the storage pool " + "(sent on create; only needed where the gateway asks for it)."), + OptionDoc("auth", "'token' (3.x /api/login), 'oauth' (4.x /rest/auth/login) " + "or 'auto' (try oauth, fall back to token).", default="auto"), + OptionDoc("sdc_naming", "'claim': name unnamed SDCs after the host and treat an " + "SDC named differently as another host's; 'ignore': never read or " + "write SDC names.", default="claim"), + ) + + def __init__(self, ctx): + super().__init__(ctx) + if self.option("auth") not in _AUTH_MODES: + raise ArrayError(f"PowerFlex option auth must be one of {', '.join(_AUTH_MODES)}") + if self.option("sdc_naming") not in _NAMING_MODES: + raise ArrayError( + f"PowerFlex option sdc_naming must be one of {', '.join(_NAMING_MODES)}") + self._mode = "" + self._refresh_token = "" + + # -- session ------------------------------------------------------------ + + def _reset_auth(self) -> None: + self.session.clear_auth() + self.session.headers.pop("Authorization", None) + self._mode = "" + self._refresh_token = "" + + def login(self) -> None: + self._reset_auth() + mode = self.option("auth") + user, password = self.secret("username"), self.secret("password") + if mode in ("auto", "oauth"): + resp = self.session.post("/rest/auth/login", + json={"username": user, "password": password}, + ok=(404, 405) if mode == "auto" else ()) + if resp.status_code < 300: + body = json_of(resp) or {} + token = body.get("access_token") if isinstance(body, dict) else None + if not token: + raise ArrayRequestError("PowerFlex /rest/auth/login returned no access_token") + self.session.headers["Authorization"] = f"Bearer {token}" + self._refresh_token = str(body.get("refresh_token") or "") + self._mode = "oauth" + return + # 404/405 under "auto": a 3.x gateway without PowerFlex Manager. + resp = self.session.get("/api/login", auth=(user, password)) + token = (resp.text or "").strip().strip('"') + if not token: + raise ArrayRequestError("PowerFlex /api/login returned no token") + self.session.set_basic_auth(user, token) + self._mode = "token" + + def close(self) -> None: + try: + if self._mode == "token": + self.session.get("/api/logout") + elif self._mode == "oauth": + self.session.post("/rest/auth/logout", + json={"refresh_token": self._refresh_token}) + except Exception: + pass + self._mode = "" + + # -- request helpers ----------------------------------------------------- + + def _req(self, method: str, path: str, *, json: Any = None, + missing_ok: bool = False) -> Any: + """Issue a request; returns the decoded body, or ``None`` for a + missing object when ``missing_ok``.""" + resp = self.session.request(method, path, json=json, ok=(400, 404, 500)) + if resp.status_code < 300: + if not resp.content: + return None + try: + return resp.json() + except ValueError: + return resp.text + try: + body = resp.json() + except ValueError: + body = None + msg = _message(body, body_text(resp)) + lowered = msg.lower() + if resp.status_code == 404 or any(s in lowered for s in _NOT_FOUND): + if missing_ok: + return None + raise ArrayNotFoundError(f"{method} {path}: {msg}") + if any(s in lowered for s in _CONFLICT): + raise ArrayConflictError(f"{method} {path}: {msg}") + raise ArrayRequestError(f"{method} {path} -> HTTP {resp.status_code}: {msg}", + status=resp.status_code) + + def _raw_volume(self, volume_id: str, missing_ok: bool = False) -> dict[str, Any] | None: + body = self._req("GET", f"/api/instances/Volume::{volume_id}", missing_ok=missing_ok) + if body is None: + return None + if not isinstance(body, dict): + raise ArrayRequestError(f"PowerFlex returned no volume for {volume_id}") + return body + + @staticmethod + def _volume(v: dict[str, Any]) -> Volume: + return Volume(id=str(v["id"]), name=v.get("name") or "", + size_bytes=int(v["sizeInKb"]) * KIB) + + def _action(self, volume_id: str, action: str, body: dict[str, Any]) -> Any: + return self._req("POST", f"/api/instances/Volume::{volume_id}/action/{action}", + json=body) + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + systems = self._req("GET", "/api/types/System/instances") or [] + system = systems[0] if isinstance(systems, list) and systems else {} + return ArrayInfo(vendor=self.vendor, model="PowerFlex", + version=str(system.get("systemVersionName") or ""), + serial=str(system.get("id") or "")) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + body: dict[str, Any] = { + "name": name, + "volumeSizeInKb": str(size_bytes // KIB), + "storagePoolId": self.option("storage_pool_id"), + "volumeType": "ThinProvisioned", + } + if self.option("protection_domain_id"): + body["protectionDomainId"] = self.option("protection_domain_id") + created = self._req("POST", "/api/types/Volume/instances", json=body) + vid = created.get("id") if isinstance(created, dict) else None + if not vid: + raise ArrayRequestError("PowerFlex volume create returned no id") + raw = self._raw_volume(vid) + assert raw is not None + return self._volume(raw) + + def get_volume(self, volume_id: str) -> Volume | None: + raw = self._raw_volume(volume_id, missing_ok=True) + return self._volume(raw) if raw else None + + def find_volume(self, name: str) -> Volume | None: + vid = self._req("POST", "/api/types/Volume/instances/action/queryIdByKey", + json={"name": name}, missing_ok=True) + if not vid or not isinstance(vid, str): + return None + return self.get_volume(vid) + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + size_gb = -(-size_bytes // GIB) + self._action(volume_id, "setVolumeSize", {"sizeInGB": str(size_gb)}) + raw = self._raw_volume(volume_id) + assert raw is not None + return self._volume(raw) + + def delete_volume(self, volume_id: str) -> None: + self._action(volume_id, "removeVolume", {"removeMode": "ONLY_ME"}) + + # -- hosts (SDCs) ----------------------------------------------------------- + + def _sdcs(self) -> list[dict[str, Any]]: + body = self._req("GET", "/api/types/Sdc/instances") + return list(body) if isinstance(body, list) else [] + + def ensure_host(self, spec: HostSpec) -> str: + guids = spec.addresses(Protocol.SDC) + if len(guids) != 1 or len(spec.initiators) != 1: + raise ArrayUnsupportedError( + "A PowerFlex host is one SDC: give exactly one SDC GUID per host") + guid = guids[0] + sdc = next((s for s in self._sdcs() + if str(s.get("sdcGuid") or "").lower() == guid.lower()), None) + if sdc is None: + raise ArrayNotFoundError( + f"No SDC with GUID {guid} is registered with PowerFlex; install the SDC " + "on the host and connect it to the MDM first") + sdc_id = str(sdc["id"]) + if self.option("sdc_naming") == "claim": + current = sdc.get("name") or "" + if current and current != spec.name: + raise ArrayConflictError( + f"SDC {guid} is registered as host {current!r}, not {spec.name!r}") + if not current: + self._req("POST", f"/api/instances/Sdc::{sdc_id}/action/setSdcName", + json={"sdcName": spec.name}) + return sdc_id + + def delete_host(self, host_id: str) -> None: + """No-op: SDCs belong to the host's PowerFlex installation.""" + + # -- mappings --------------------------------------------------------------- + + def list_mappings(self, volume_id: str) -> list[Mapping]: + raw = self._raw_volume(volume_id) + assert raw is not None + info = raw.get("mappedSdcInfo") or [] # null when unmapped + return [Mapping(volume_id=volume_id, host_id=str(m["sdcId"]), lun=None) + for m in info if m.get("sdcId")] + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + for m in self.list_mappings(volume_id): + if m.host_id == host_id: + return m + self._action(volume_id, "addMappedSdc", { + "sdcId": host_id, "allowMultipleMappings": "TRUE", "accessMode": "ReadWrite"}) + return Mapping(volume_id=volume_id, host_id=host_id, lun=None) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + raw = self._raw_volume(volume_id, missing_ok=True) + if raw is None: + return + if not any(str(m.get("sdcId")) == host_id for m in raw.get("mappedSdcInfo") or []): + return + try: + self._action(volume_id, "removeMappedSdc", {"sdcId": host_id}) + except ArrayRequestError as e: + if not any(s in str(e).lower() for s in _NOT_MAPPED): + raise diff --git a/simplyblock_core/storage_arrays/adapters/dell_powermax.py b/simplyblock_core/storage_arrays/adapters/dell_powermax.py new file mode 100644 index 0000000000..a05fbbd1fd --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/dell_powermax.py @@ -0,0 +1,434 @@ +"""Dell PowerMax, Unisphere for PowerMax REST API (10.x). + +Auth: HTTP basic on every request (no session). Provisioning lives under +``/univmax/restapi/{api_version}/sloprovisioning/symmetrix/{symmetrix_id}``. +Errors are ``{"message": "..."}``, mostly with status 400; 404 for missing +objects; 401 for a rejected credential. + +Layout (one object set per simplyblock volume): + +* storage group ```` (``sb-``) in the configured SRP / + service level, holding exactly one device; +* the device's ``volume_identifier`` is the same name; +* one masking view per (storage group, host): ``_`` (or + ``_`` when that exceeds 64 characters), joining the storage + group, the host (initiator group) and the port group configured for the + host's protocol. + +A storage group per volume keeps map/unmap/delete independent per volume +(PowerMax maps storage groups, not devices), at the cost of one SG and one +masking view per mapping; arrays allow tens of thousands of each. + +``Volume.id`` is the 5-hex-digit device id (``0012A``). ``Volume.wwn`` is +``naa.`` + the device's ``effective_wwn`` (falls back to ``wwn``). Volume +identifiers are *not* unique on PowerMax; lookups by name prefer the device +in the storage group of that name and refuse to guess between several. + +Sizes: devices are allocated in cylinders; on PowerMaxOS (FBA) one cylinder +is 15 tracks x 128 KiB = 1920 KiB, so the size granularity is 1 966 080 +bytes and all sizes are sent as ``capacityUnit: "CYL"`` (exact, no GB/MB +rounding). ``cap_cyl`` is read back. + +Async jobs: with ``async_jobs`` the adapter sends +``executionOption: ASYNCHRONOUS`` and polls ``/{ver}/system/job/{jobId}`` on +``202 Accepted``; otherwise it sends ``SYNCHRONOUS`` but still follows a 202 +should the server answer with a job. + +Verify against a real system +---------------------------- +* Whether ``sloId: "None"`` (no service level) is accepted on 10.x, and that + ``workloadSelection`` may be omitted. +* The empty answer of ``GET .../volume?volume_identifier=x`` (assumed: + ``count: 0`` with no ``resultList``) and whether the filter is exact or + case-insensitive; iterator paging (``/common/Iterator``) is not followed -- + fine for name lookups that match one device. +* The ``storageGroupId`` filter on ``GET .../volume``. +* Status codes and wording for: SG / host / masking view name already in use, + initiator already in another initiator group (classified by "already" / + "in use" / "exist" / "member of"), removing the last device of a masked SG, + deleting a device still in an SG, shrink. +* Whether a device must be removed from its SG before ``DELETE volume`` (done + here) and whether 10.x still needs a deallocation step first (not done). +* Whether iSCSI IQNs / NVMe host NQNs must already be logged in to the array + before ``POST host`` accepts them (FC WWNs need not be). +* NVMe/TCP: initiator ids for NVMe hosts (assumed: the host NQN) and whether + the volume object carries an ``nguid`` field (read if present). +* ``GET maskingview/{mv}/connections?volume_id=`` returns rows only for + initiator/port pairs (assumed: one row per initiator x port, logged in or + not); ``host_lun_address`` is hex. With no rows the mapping's LUN is None. +* Job states (``SUCCEEDED`` / ``FAILED`` / ``VALIDATE_FAILED``; others are + in progress) and that the failure text is in ``result``. +""" +from __future__ import annotations + +import hashlib +import math +from collections.abc import Callable +from typing import Any + +import requests + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import json_of + +CYLINDER_BYTES = 1920 * 1024 +MAX_NAME = 64 +_JOB_DONE = "SUCCEEDED" +_JOB_FAILED = frozenset({"FAILED", "VALIDATE_FAILED"}) +_JOB_POLL_S = 2.0 + +_NOT_FOUND = ("cannot find", "not found", "does not exist", "no such") +_CONFLICT = ("already", "in use", "exists", "member of", "is part of", "masking view") + + +def _message(resp: requests.Response) -> str: + try: + body = json_of(resp) + except ArrayRequestError: + body = None + if isinstance(body, dict) and body.get("message"): + return str(body["message"]) + return (resp.text or "")[:600] or f"HTTP {resp.status_code}" + + +def _classify(what: str, text: str, status: int | None = None) -> ArrayError: + lowered = text.lower() + msg = f"{what}: {text}" + if status == 404 or any(k in lowered for k in _NOT_FOUND): + return ArrayNotFoundError(msg) + if any(k in lowered for k in _CONFLICT): + return ArrayConflictError(msg) + return ArrayRequestError(msg, status=status) + + +def powermax_wwn(item: dict[str, Any]) -> str | None: + v = str(item.get("effective_wwn") or item.get("wwn") or "").strip().lower() + v = v.removeprefix("naa.") + return f"naa.{v}" if v else None + + +def initiator_protocol(initiator_id: str) -> Protocol: + """Protocol of an initiator-group member as Unisphere lists it.""" + v = initiator_id.lower() + if v.startswith(("iqn.", "eui.")): + return Protocol.ISCSI + if v.startswith("nqn."): + return Protocol.NVME_TCP + return Protocol.FC + + +class DellPowerMaxAdapter(ArrayAdapter): + vendor = "dell-powermax" + display_name = "Dell PowerMax" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC, Protocol.NVME_TCP}), + size_granularity_bytes=CYLINDER_BYTES, + min_size_bytes=CYLINDER_BYTES, + max_name_len=MAX_NAME, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("symmetrix_id", "12-digit array serial (symmetrix id), e.g. '000197900123'.", + required=True), + OptionDoc("srp", "Storage resource pool for new storage groups.", default="SRP_1"), + OptionDoc("service_level", "Service level (e.g. 'Diamond'); default: none."), + OptionDoc("port_group", "Port group for masking views: a name, or a map from " + "protocol ('iscsi', 'fc', 'nvme-tcp') to name. Needed to map volumes."), + OptionDoc("api_version", "Unisphere REST version in the URL ('100' = 10.0, " + "'101' = 10.1, ...).", default="100"), + OptionDoc("async_jobs", "Run modifying calls as Unisphere jobs and poll them " + "(recommended for large arrays where synchronous calls time out).", + default=False), + OptionDoc("job_timeout_s", "How long to wait for an asynchronous job.", default=600), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._base = f"/univmax/restapi/{self.option('api_version')}" + self._sym = str(self.option("symmetrix_id")) + self._slo = f"{self._base}/sloprovisioning/symmetrix/{self._sym}" + self._sleep: Callable[[float], None] = self.session.sleep + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.set_basic_auth(self.secret("username"), self.secret("password")) + + def close(self) -> None: + self.session.clear_auth() + + # -- request helpers ----------------------------------------------------- + + def _exec(self) -> str: + return "ASYNCHRONOUS" if self.option("async_jobs") else "SYNCHRONOUS" + + def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None, missing_ok: bool = False) -> Any: + resp = self.session.request(method, path, params=params, json=json, ok=(400, 404)) + if resp.status_code in (400, 404): + err = _classify(f"{method} {path} -> HTTP {resp.status_code}", _message(resp), + resp.status_code) + if missing_ok and isinstance(err, ArrayNotFoundError): + return None + raise err + body = json_of(resp) + if resp.status_code == 202 and isinstance(body, dict) and body.get("jobId"): + return self._wait_job(str(body["jobId"]), f"{method} {path}") + return body + + def _wait_job(self, job_id: str, what: str) -> dict[str, Any]: + polls = max(1, math.ceil(float(self.option("job_timeout_s")) / _JOB_POLL_S)) + for _ in range(polls): + job = json_of(self.session.get(f"{self._base}/system/job/{job_id}")) or {} + status = str(job.get("status", "")).upper() + if status == _JOB_DONE: + return job + if status in _JOB_FAILED: + raise _classify(f"{what} (job {job_id} {status})", + str(job.get("result") or "no result")) + self._sleep(_JOB_POLL_S) + raise ArrayRequestError(f"{what}: job {job_id} did not finish within " + f"{self.option('job_timeout_s')} s") + + @staticmethod + def _volume(item: dict[str, Any]) -> Volume: + nguid = str(item.get("nguid") or "").strip().lower().removeprefix("eui.") + return Volume(id=item["volumeId"], name=item.get("volume_identifier") or "", + size_bytes=int(item["cap_cyl"]) * CYLINDER_BYTES, + wwn=powermax_wwn(item), nguid=f"eui.{nguid}" if nguid else None) + + def _volume_item(self, device_id: str) -> dict[str, Any] | None: + return self._call("GET", f"{self._slo}/volume/{device_id}", missing_ok=True) + + def _require_item(self, device_id: str) -> dict[str, Any]: + item = self._volume_item(device_id) + if item is None: + raise ArrayNotFoundError(f"PowerMax device {device_id} not found") + return item + + def _device_ids(self, **filters: str) -> list[str]: + body = self._call("GET", f"{self._slo}/volume", params=filters) or {} + rows = (body.get("resultList") or {}).get("result") or [] + return [r["volumeId"] for r in rows if r.get("volumeId")] + + @staticmethod + def _cyl(size_bytes: int) -> int: + return -(-int(size_bytes) // CYLINDER_BYTES) + + def _attrs(self, name: str, size_bytes: int) -> dict[str, Any]: + return {"num_of_vols": 1, "volume_size": str(self._cyl(size_bytes)), + "capacityUnit": "CYL", + "volumeIdentifier": {"identifier_name": name, + "volumeIdentifierChoice": "identifier_name"}} + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + about = self._call("GET", "/univmax/restapi/version") or {} + supported = about.get("supported_api_versions") + if supported and str(self.option("api_version")) not in [str(v) for v in supported]: + raise ArrayUnsupportedError( + f"Unisphere {about.get('version', '')} does not serve REST version " + f"{self.option('api_version')} (supported: {', '.join(map(str, supported))})") + sym = self._call("GET", f"{self._base}/system/symmetrix/{self._sym}") or {} + version = sym.get("ucode") or "" + unisphere = about.get("version") or "" + return ArrayInfo(vendor=self.vendor, model=sym.get("model", ""), + version=f"PowerMaxOS {version} (Unisphere {unisphere})".strip(), + serial=sym.get("symmetrixId", self._sym)) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + attrs = self._attrs(name, size_bytes) + payload: dict[str, Any] = { + "srpId": self.option("srp"), + "storageGroupId": name, + "emulation": "FBA", + "sloBasedStorageGroupParam": [{ + "sloId": self.option("service_level") or "None", + "volumeAttributes": [attrs], + }], + "executionOption": self._exec(), + } + try: + self._call("POST", f"{self._slo}/storagegroup", json=payload) + except ArrayConflictError: + # Left over from an interrupted create or delete: reuse it if empty. + sg = self._call("GET", f"{self._slo}/storagegroup/{name}", missing_ok=True) + if sg is None or int(sg.get("num_of_vols") or 0): + raise + self._call("PUT", f"{self._slo}/storagegroup/{name}", json={ + "editStorageGroupActionParam": {"expandStorageGroupParam": {"addVolumeParam": { + "create_new_volumes": True, "emulation": "FBA", + "volumeAttributes": [attrs]}}}, + "executionOption": self._exec()}) + devices = self._device_ids(storageGroupId=name) + if len(devices) != 1: + raise ArrayRequestError( + f"PowerMax storage group {name} holds {len(devices)} devices after create") + return self._volume(self._require_item(devices[0])) + + def get_volume(self, volume_id: str) -> Volume | None: + item = self._volume_item(volume_id) + return self._volume(item) if item else None + + def find_volume(self, name: str) -> Volume | None: + items = [i for i in (self._volume_item(d) for d in self._device_ids(volume_identifier=name)) + if i and i.get("volume_identifier") == name] + if len(items) > 1: + own = [i for i in items if name in (i.get("storageGroupId") or [])] + if len(own) != 1: + raise ArrayConflictError( + f"{len(items)} PowerMax devices are named {name}: " + + ", ".join(i["volumeId"] for i in items)) + items = own + return self._volume(items[0]) if items else None + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + self._call("PUT", f"{self._slo}/volume/{volume_id}", json={ + "editVolumeActionParam": {"expandVolumeParam": {"volumeAttribute": { + "volume_size": str(self._cyl(size_bytes)), "capacityUnit": "CYL"}}}, + "executionOption": self._exec()}) + return self._volume(self._require_item(volume_id)) + + def delete_volume(self, volume_id: str) -> None: + item = self._require_item(volume_id) + name = item.get("volume_identifier") or "" + for sg in item.get("storageGroupId") or []: + self._call("PUT", f"{self._slo}/storagegroup/{sg}", json={ + "editStorageGroupActionParam": {"removeVolumeParam": {"volumeId": [volume_id]}}, + "executionOption": self._exec()}) + if sg == name: + self._call("DELETE", f"{self._slo}/storagegroup/{sg}", missing_ok=True) + self._call("DELETE", f"{self._slo}/volume/{volume_id}") + + # -- hosts ------------------------------------------------------------------ + + def ensure_host(self, spec: HostSpec) -> str: + wanted = [i.normalized() for i in spec.initiators] + host = self._call("GET", f"{self._slo}/host/{spec.name}", missing_ok=True) + if host is None: + self._call("POST", f"{self._slo}/host", json={ + "hostId": spec.name, "initiatorId": wanted, "executionOption": self._exec()}) + return spec.name + have = {str(i).lower() for i in host.get("initiator") or []} + missing = [w for w in wanted if w.lower() not in have] + if missing: + self._call("PUT", f"{self._slo}/host/{spec.name}", json={ + "editHostActionParam": {"addInitiatorParam": {"initiator": missing}}, + "executionOption": self._exec()}) + return spec.name + + def delete_host(self, host_id: str) -> None: + self._call("DELETE", f"{self._slo}/host/{host_id}", missing_ok=True) + + # -- mappings ------------------------------------------------------------------ + + @staticmethod + def view_name(sg: str, host: str) -> str: + name = f"{sg}_{host}" + if len(name) <= MAX_NAME: + return name + return f"{sg}_{hashlib.sha256(host.encode()).hexdigest()[:16]}" + + def _port_group(self, host_id: str) -> str: + option = self.option("port_group") + if not option: + raise ArrayError(f"{self.display_name} array {self.config.id} needs the " + "'port_group' option to map volumes") + if isinstance(option, str): + return option + host = self._call("GET", f"{self._slo}/host/{host_id}") + protocols = {initiator_protocol(str(i)) for i in host.get("initiator") or []} + if len(protocols) != 1: + raise ArrayError(f"Cannot tell the protocol of PowerMax host {host_id}") + protocol = protocols.pop() + pg = option.get(protocol.value) + if not pg: + raise ArrayError(f"No PowerMax port group configured for {protocol.value} " + f"(option 'port_group')") + return str(pg) + + def _views(self, item: dict[str, Any]) -> list[tuple[str, str, str]]: + """(masking view, host or host group, storage group) for a device.""" + out = [] + for sg in item.get("storageGroupId") or []: + sg_item = self._call("GET", f"{self._slo}/storagegroup/{sg}", missing_ok=True) + for mv in (sg_item or {}).get("maskingview") or []: + mv_item = self._call("GET", f"{self._slo}/maskingview/{mv}", missing_ok=True) + if mv_item: + host = mv_item.get("hostId") or mv_item.get("hostGroupId") + if host: + out.append((mv, str(host), sg)) + return out + + def _lun(self, view: str, device_id: str) -> int | None: + body = self._call("GET", f"{self._slo}/maskingview/{view}/connections", + params={"volume_id": device_id}, missing_ok=True) or {} + for row in body.get("maskingViewConnection") or []: + if row.get("volumeId") == device_id and row.get("host_lun_address"): + return int(str(row["host_lun_address"]), 16) + return None + + def _mappings(self, item: dict[str, Any]) -> list[Mapping]: + seen: dict[str, Mapping] = {} + for view, host, _sg in self._views(item): + if host not in seen: + seen[host] = Mapping(volume_id=item["volumeId"], host_id=host, + lun=self._lun(view, item["volumeId"])) + return list(seen.values()) + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + item = self._require_item(volume_id) + for m in self._mappings(item): + if m.host_id == host_id: + return m + sg = item.get("volume_identifier") or "" + if sg not in (item.get("storageGroupId") or []): + raise ArrayConflictError( + f"PowerMax device {volume_id} is not in its own storage group {sg!r}") + if self._call("GET", f"{self._slo}/host/{host_id}", missing_ok=True) is None: + raise ArrayNotFoundError(f"PowerMax host {host_id} not found") + self._call("POST", f"{self._slo}/maskingview", json={ + "maskingViewId": self.view_name(sg, host_id), + "hostOrHostGroupSelection": {"useExistingHostParam": {"hostId": host_id}}, + "portGroupSelection": {"useExistingPortGroupParam": { + "portGroupId": self._port_group(host_id)}}, + "storageGroupSelection": {"useExistingStorageGroupParam": {"storageGroupId": sg}}, + "executionOption": self._exec()}) + for m in self._mappings(self._require_item(volume_id)): + if m.host_id == host_id: + return m + raise ArrayRequestError( + f"PowerMax masked device {volume_id} to host {host_id} but lists no mapping") + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + item = self._volume_item(volume_id) + if item is None: + return + own = item.get("volume_identifier") or "" + for view, host, sg in self._views(item): + if host != host_id: + continue + if sg != own: + raise ArrayConflictError( + f"PowerMax device {volume_id} is mapped to {host_id} through shared " + f"storage group {sg} (masking view {view}); unmap it on the array") + self._call("DELETE", f"{self._slo}/maskingview/{view}", missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + item = self._volume_item(volume_id) + return self._mappings(item) if item else [] diff --git a/simplyblock_core/storage_arrays/adapters/dell_powerstore.py b/simplyblock_core/storage_arrays/adapters/dell_powerstore.py new file mode 100644 index 0000000000..4a480d92e4 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/dell_powerstore.py @@ -0,0 +1,332 @@ +"""Dell PowerStore, REST API (PowerStoreOS 3.x / 4.x). + +Auth: HTTP basic on ``GET /api/rest/login_session`` opens a session. The +response carries the session cookie (``auth_cookie``) and a CSRF token in the +``DELL-EMC-TOKEN`` header; every later request sends the cookie, and every +non-GET request must also send the token header (the array answers 403 +without it). Basic auth is *not* kept on the session afterwards, so an expired +session surfaces as 401 and :class:`ArrayClient` logs in again. + +Queries use the PostgREST-style syntax: ``select=a,b`` picks fields, +``name=eq.x`` filters, ``limit``/``offset`` page; a partial page is answered +with ``206 Partial Content`` and a ``Content-Range`` header. + +Errors: ``{"messages": [{"code": "0x...", "severity": "Error", +"message_l10n": "...", "arguments": [...]}]}``. Semantic failures (name in +use, initiator owned by another host, volume still attached, shrink) come +back as ``422 Unprocessable Entity``; malformed requests as 400. + +Objects are addressed by id (UUID) for volumes and hosts. Volume identity: +``wwn`` is returned as ``naa.``, ``nguid`` as ``nguid.<32 hex>``; this +adapter reports them as ``naa.`` / ``eui.`` (lowercase). + +Sizes are bytes; PowerStore requires a multiple of 8 KiB (8192) and at least +1 MiB. + +Verify against a real system +---------------------------- +* The session cookie name (``auth_cookie``) and that the CSRF header is + exactly ``DELL-EMC-TOKEN`` on both the login response and requests. +* That ``POST /volume`` and ``POST /host`` answer ``201 {"id": ...}`` and that + PATCH/DELETE/attach/detach answer ``204`` with an empty body. +* The status code (422 assumed) and ``message_l10n`` wording for: duplicate + name, initiator already registered to another host, delete of an attached + volume/host, shrink, detach of a non-attached volume. Classification below + is by wording ("already", "in use", "attached", "mapped", "not found"). +* The host field holding initiators on GET is ``host_initiators`` + (create uses ``initiators``, modify ``add_initiators``), each + ``{"port_name", "port_type"}``; ``port_type`` values ``iSCSI``/``FC``/ + ``NVMe`` (NVMe/TCP and NVMe/FC both ``NVMe``), and that FC ``port_name`` is + accepted/returned colon-separated lowercase (``21:00:...``). +* That ``logical_unit_number`` in ``host_volume_mapping`` is the NVMe NSID for + NVMe hosts, and which LUN the array picks first (fake: 1). +* That a ``name=eq.`` filter needs no quoting for host names containing + dots or commas. +* That the size multiple is 8192 bytes on 4.x as on 3.x. +""" +from __future__ import annotations + +from http.cookies import CookieError, SimpleCookie +from typing import Any + +import requests + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, + colon_wwpn, + normalize_wwpn, +) +from ..http import json_of + +API = "/api/rest" +TOKEN_HEADER = "DELL-EMC-TOKEN" +VOLUME_FIELDS = "id,name,size,wwn,nguid" +HOST_FIELDS = "id,name,host_initiators" +MAPPING_FIELDS = "id,host_id,host_group_id,volume_id,logical_unit_number" +_PAGE = 1000 + +_NOT_FOUND = ("not found", "does not exist", "not exist", "no such") +_CONFLICT = ("already", "in use", "exists", "attached", "mapped", "is used", "duplicate") + + +def _messages(resp: requests.Response) -> str: + try: + body = json_of(resp) or {} + except ArrayRequestError: + return "" + if not isinstance(body, dict): + return "" + return " ".join(str(m.get("message_l10n") or m.get("code") or "") + for m in body.get("messages", []) if isinstance(m, dict)) + + +def _classify(method: str, path: str, resp: requests.Response) -> ArrayError: + text = _messages(resp) or f"HTTP {resp.status_code}" + lowered = text.lower() + msg = f"{method} {path} -> HTTP {resp.status_code}: {text}" + if resp.status_code == 404 or any(k in lowered for k in _NOT_FOUND): + return ArrayNotFoundError(msg) + if any(k in lowered for k in _CONFLICT): + return ArrayConflictError(msg) + return ArrayRequestError(msg, status=resp.status_code) + + +def powerstore_wwn(value: str | None) -> str | None: + v = (value or "").strip().lower().removeprefix("naa.") + return f"naa.{v}" if v else None + + +def powerstore_nguid(value: str | None) -> str | None: + v = (value or "").strip().lower().removeprefix("nguid.").removeprefix("eui.") + return f"eui.{v}" if v else None + + +_PORT_TYPE = { + Protocol.ISCSI: "iSCSI", + Protocol.FC: "FC", + Protocol.NVME_TCP: "NVMe", + Protocol.NVME_FC: "NVMe", +} + + +def _port_key(port_type: str, port_name: str) -> str: + """Comparable form of an initiator as the array reports it.""" + if port_type == "FC": + try: + return normalize_wwpn(port_name) + except ValueError: + return port_name.lower() + if port_type == "iSCSI": + return port_name.lower() + return port_name + + +class DellPowerStoreAdapter(ArrayAdapter): + vendor = "dell-powerstore" + display_name = "Dell PowerStore" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC, Protocol.NVME_TCP, Protocol.NVME_FC}), + size_granularity_bytes=8192, + min_size_bytes=1 << 20, + max_name_len=128, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("os_type", "os_type of hosts simplyblock creates.", default="Linux"), + OptionDoc("appliance_id", "Create volumes on this appliance of a multi-appliance " + "cluster (e.g. 'A1'); default: the array places them."), + ) + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.headers.pop(TOKEN_HEADER, None) + self.session.http.cookies.clear() + resp = self.session.get(f"{API}/login_session", + auth=(self.secret("username"), self.secret("password"))) + token = resp.headers.get(TOKEN_HEADER) + if not token: + raise ArrayRequestError(f"PowerStore login returned no {TOKEN_HEADER} header") + self.session.headers[TOKEN_HEADER] = token + self._adopt_cookies(resp) + + def _adopt_cookies(self, resp: requests.Response) -> None: + """Keep the session cookie. ``requests`` normally stores it already; + this also covers transports that do not populate the cookie jar.""" + raw = resp.headers.get("Set-Cookie") + if not raw: + return + jar = self.session.http.cookies + try: + parsed = SimpleCookie() + parsed.load(raw) + except CookieError: + return + for name, morsel in parsed.items(): + if name not in jar: + jar.set(name, morsel.value, path=morsel["path"] or "/") + + def close(self) -> None: + if TOKEN_HEADER in self.session.headers: + try: + self.session.post(f"{API}/logout", ok=(401, 403)) + except Exception: + pass + self.session.headers.pop(TOKEN_HEADER, None) + + # -- request helpers ----------------------------------------------------- + + def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None, missing_ok: bool = False) -> requests.Response | None: + resp = self.session.request(method, f"{API}{path}", params=params, json=json, + ok=(400, 404, 422)) + if resp.status_code in (400, 404, 422): + err = _classify(method, path, resp) + if missing_ok and isinstance(err, ArrayNotFoundError): + return None + raise err + return resp + + def _one(self, path: str, select: str) -> dict[str, Any] | None: + resp = self._call("GET", path, params={"select": select}, missing_ok=True) + return json_of(resp) if resp is not None else None + + def _list(self, path: str, select: str, **filters: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + offset = 0 + while True: + params: dict[str, Any] = {"select": select, **{k: f"eq.{v}" for k, v in filters.items()}, + "limit": _PAGE, "offset": offset} + resp = self.session.request("GET", f"{API}{path}", params=params, + ok=(206, 400, 422)) + if resp.status_code in (400, 422): + raise _classify("GET", path, resp) + items = json_of(resp) or [] + out.extend(items) + if resp.status_code != 206 or not items: + return out + offset += len(items) + + @staticmethod + def _volume(item: dict[str, Any]) -> Volume: + return Volume(id=item["id"], name=item["name"], size_bytes=int(item["size"]), + wwn=powerstore_wwn(item.get("wwn")), + nguid=powerstore_nguid(item.get("nguid"))) + + def _require_volume(self, volume_id: str) -> Volume: + vol = self.get_volume(volume_id) + if vol is None: + raise ArrayNotFoundError(f"PowerStore volume {volume_id} not found") + return vol + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + clusters = self._list("/cluster", "id,name,global_id") + cluster = clusters[0] if clusters else {} + model = version = "" + try: + appliances = self._list("/appliance", "id,name,model") + model = next((a.get("model", "") for a in appliances if a.get("model")), "") + software = self._list("/software_installed", "id,release_version,is_cluster", + is_cluster="true") + version = next((s.get("release_version", "") for s in software), "") + except (ArrayRequestError, ArrayNotFoundError): + pass + return ArrayInfo(vendor=self.vendor, model=model, + version=f"PowerStoreOS {version}".strip(), + serial=cluster.get("global_id") or cluster.get("id", "")) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + body: dict[str, Any] = {"name": name, "size": size_bytes} + if self.option("appliance_id"): + body["appliance_id"] = self.option("appliance_id") + resp = self._call("POST", "/volume", json=body) + created = json_of(resp) if resp is not None else None + if not created or "id" not in created: + raise ArrayRequestError(f"PowerStore volume create for {name} returned no id") + return self._require_volume(created["id"]) + + def get_volume(self, volume_id: str) -> Volume | None: + item = self._one(f"/volume/{volume_id}", VOLUME_FIELDS) + return self._volume(item) if item else None + + def find_volume(self, name: str) -> Volume | None: + items = self._list("/volume", VOLUME_FIELDS, name=name) + return self._volume(items[0]) if items else None + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + self._call("PATCH", f"/volume/{volume_id}", json={"size": size_bytes}) + return self._require_volume(volume_id) + + def delete_volume(self, volume_id: str) -> None: + self._call("DELETE", f"/volume/{volume_id}") + + def _host_by_name(self, name: str) -> dict[str, Any] | None: + items = self._list("/host", HOST_FIELDS, name=name) + return items[0] if items else None + + def ensure_host(self, spec: HostSpec) -> str: + wanted: list[dict[str, str]] = [] + for i in spec.initiators: + port_name = colon_wwpn(i.address) if i.protocol == Protocol.FC else i.normalized() + wanted.append({"port_name": port_name, "port_type": _PORT_TYPE[i.protocol]}) + host = self._host_by_name(spec.name) + if host is None: + resp = self._call("POST", "/host", json={ + "name": spec.name, "os_type": self.option("os_type"), "initiators": wanted}) + created = json_of(resp) if resp is not None else None + if not created or "id" not in created: + raise ArrayRequestError(f"PowerStore host create for {spec.name} returned no id") + return str(created["id"]) + have = {_port_key(p.get("port_type", ""), p.get("port_name", "")) + for p in host.get("host_initiators") or []} + missing = [w for w in wanted if _port_key(w["port_type"], w["port_name"]) not in have] + if missing: + self._call("PATCH", f"/host/{host['id']}", json={"add_initiators": missing}) + return str(host["id"]) + + def delete_host(self, host_id: str) -> None: + self._call("DELETE", f"/host/{host_id}", missing_ok=True) + + def _mappings(self, volume_id: str) -> list[Mapping]: + items = self._list("/host_volume_mapping", MAPPING_FIELDS, volume_id=volume_id) + out = [] + for i in items: + host = i.get("host_id") or i.get("host_group_id") + if host: + lun = i.get("logical_unit_number") + out.append(Mapping(volume_id=i["volume_id"], host_id=host, + lun=int(lun) if lun is not None else None)) + return out + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + for m in self._mappings(volume_id): + if m.host_id == host_id: + return m + self._call("POST", f"/volume/{volume_id}/attach", json={"host_id": host_id}) + for m in self._mappings(volume_id): + if m.host_id == host_id: + return m + raise ArrayRequestError( + f"PowerStore attached volume {volume_id} to host {host_id} but lists no mapping") + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + if not any(m.host_id == host_id for m in self._mappings(volume_id)): + return + self._call("POST", f"/volume/{volume_id}/detach", json={"host_id": host_id}, + missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return self._mappings(volume_id) diff --git a/simplyblock_core/storage_arrays/adapters/hitachi_vsp.py b/simplyblock_core/storage_arrays/adapters/hitachi_vsp.py new file mode 100644 index 0000000000..29cead34cd --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/hitachi_vsp.py @@ -0,0 +1,570 @@ +"""Hitachi VSP (VSP One Block, VSP E series, VSP 5000, VSP G/F) through the +Configuration Manager REST API. + +Deployment modes (option ``storage_device_id``): + +* embedded (option absent): the API runs on the SVP / controller and objects + live under ``/ConfigurationManager/v1/objects/...``; +* REST API server (Ops Center API Configuration Manager): the storage system + must already be registered on the server; objects live under + ``/ConfigurationManager/v1/objects/storages/{storageDeviceId}/...``. + +Auth: ``POST .../sessions`` with HTTP basic auth (a storage-system user) +returns ``{"token", "sessionId"}``; later requests send ``Authorization: +Session ``; ``DELETE .../sessions/{sessionId}`` discards it. An expired +session answers 401, which the client turns into one re-login. + +Asynchronous jobs: create/modify/delete requests answer ``202`` with a job +object (``jobId``, ``status``, ``state``, ``affectedResources``, ``error``). +The adapter polls ``GET .../jobs/{jobId}`` until ``status == "Completed"``; +``state`` is then ``Succeeded`` or ``Failed`` (``error`` carries the usual +error body). Polling gives up after ``job_timeout_s``; the job may still +finish on the array, and every operation is safe to re-run. + +Object model: + +* volume = DP-VOL LDEV in pool ``pool_id``; ``Volume.id`` is the decimal LDEV + number, ``Volume.name`` the LDEV label (max 32 characters, not unique on the + array). A deleted LDEV still answers ``GET /ldevs/{id}`` with + ``emulationType == "NOT DEFINED"``, which reads as "missing". +* host = one host group (FC) / iSCSI target (iSCSI) of the same name on every + configured port of the matching type; the host id is that name. Initiators + are ``host-wwns`` / ``host-iscsis`` of each of those groups. +* mapping = one LU path per port (``POST /luns``); the adapter uses the same + LUN number on every port and reports the LUN of the first configured port. + +Volume identity: SCSI ``naa.`` (lowercase). + +Verify against a real system: + +* ``POST /ldevs`` without ``ldevId`` auto-assigns a free LDEV number (newer + API versions); older microcode may require ``ldevId`` or a + ``startLdevId``/``endLdevId`` range -- set ``ldev_range`` there. +* The label is set in a second request (``PUT /ldevs/{id}``); a crash between + the two leaves an unlabeled LDEV the client cannot find (a label failure + is cleaned up here, a crash is not). +* ``find_volume`` lists DP-VOLs of the pool (``ldevOption=dpVolume&poolId=``, + paged by ``headLdevId``/``count``) and matches labels client-side: one or a + few requests per 16k LDEVs. The API has no label filter that we know of. +* ``naaId`` is derived from the serial and LDEV number, so a deleted LDEV's + number reused by a later volume has the same WWN; ownership then rests on + the ``sb-`` label. +* DP-VOL capacity: the adapter rounds to 1 MiB (safe for data-reduction + volumes, which need MiB alignment) and creates at least 64 MiB (DP-VOLs + have a ~47 MB floor on some models). +* ``GET /ldevs/{id}`` lists its LU paths under ``ports`` (``portId``, + ``hostGroupNumber``, ``hostGroupName``, ``lun``); mappings are read from + there. +* ``GET /host-wwns`` and ``/host-iscsis`` need ``portId`` and + ``hostGroupNumber``, so detecting an initiator owned by another host lists + every host group on each configured port (one request per group). + FC rejects a WWN registered to two host groups of one port; iSCSI allows + the same initiator in several targets, so the scan is what enforces + ownership there. +* Some models / configurations require the resource-group lock + (``services/resource-group-service/actions/lock/invoke``) around changes; + enable with ``use_lock``. +* Error ``messageId``s: storage-side failures mostly arrive as + ``KART30000-E`` with SSB codes in ``errorCode``; errors are classified by + HTTP status first, then by message text (see ``_classify``). +* NVMe-oF (``/nvm-subsystems``) is not implemented: only FC and iSCSI. +""" +from __future__ import annotations + +import re +import time +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import json_of + +ROOT = "/ConfigurationManager/v1/objects" +BLOCK = 512 +NOT_DEFINED = "NOT DEFINED" +MAX_LUN = 2047 +#: ``count`` per ``GET /ldevs`` page (the API maximum). +PAGE_SIZE = 16384 + +_PORT_RE = re.compile(r"^CL[0-9A-Z]+-[A-Z0-9]+$") +_RANGE_RE = re.compile(r"^\s*(\d+)\s*-\s*(\d+)\s*$") +_PORT_TYPES = {"FIBRE": Protocol.FC, "ISCSI": Protocol.ISCSI} +#: HTTP statuses whose Hitachi error body the adapter classifies itself +#: (401/403 stay with RestSession -> ArrayAuthError). +_OWN_STATUSES = (400, 404, 405, 409, 412, 500, 502, 503, 504) + +_CONFLICT_TEXT = ("already", "in use", "is used", "registered", "duplicat", + "lu path is defined", "lu paths are defined", "locked") +_MISSING_TEXT = ("does not exist", "not found", "not defined", "not exist") + + +def _monotonic() -> float: + return time.monotonic() + + +def _classify(where: str, status: int | None, body: Any) -> ArrayError: + """Map a Hitachi error body (``errorSource``, ``message``, ``solution``, + ``messageId``, ``errorCode``) to an ArrayError.""" + if isinstance(body, dict): + message = str(body.get("message") or body.get("error") or "") + msg_id = str(body.get("messageId") or "") + codes = body.get("errorCode") + ssb = "" + if isinstance(codes, dict) and codes: + ssb = " (" + ", ".join(f"{k}={v}" for k, v in sorted(codes.items())) + ")" + text = f"{where}: {msg_id + ' ' if msg_id else ''}{message}{ssb}".strip() + else: + message = str(body or "") + text = f"{where}: {message}" + if status: + text += f" [HTTP {status}]" + lowered = message.lower() + if status == 404: + return ArrayNotFoundError(text) + if status in (409, 412): + return ArrayConflictError(text) + if status is None or status in (400, 500): + if any(t in lowered for t in _MISSING_TEXT): + return ArrayNotFoundError(text) + if any(t in lowered for t in _CONFLICT_TEXT): + return ArrayConflictError(text) + return ArrayRequestError(text, status=status) + + +def _tail(resource: str) -> str: + """``/ConfigurationManager/v1/objects/ldevs/123`` -> ``123``.""" + return resource.rstrip("/").rsplit("/", 1)[-1] + + +class HitachiVspAdapter(ArrayAdapter): + vendor = "hitachi-vsp" + display_name = "Hitachi VSP" + capabilities = Capabilities( + protocols=frozenset({Protocol.FC, Protocol.ISCSI}), + size_granularity_bytes=1 << 20, + min_size_bytes=64 << 20, + # LDEV labels and iSCSI target names: 32 characters. + max_name_len=32, + ) + option_docs = ( + OptionDoc("pool_id", "DP (HDP/HDT) pool number volumes are created in.", + required=True), + OptionDoc("ports", "Target ports hosts are presented on, e.g. " + "['CL1-A', 'CL2-A']. FC and iSCSI ports may be mixed; a host " + "uses the ports of its initiators' protocol.", required=True), + OptionDoc("storage_device_id", "storageDeviceId (e.g. '886000123456') when " + "the endpoint is an Ops Center API Configuration Manager server; " + "leave unset for the API embedded on the SVP/controller."), + OptionDoc("ldev_range", "Restrict new LDEV numbers (and volume lookups) " + "to 'start-end' (decimal LDEV ids)."), + OptionDoc("host_mode", "hostMode of new host groups.", default="LINUX/IRIX"), + OptionDoc("host_mode_options", "hostModeOptions numbers for new host groups.", + default=()), + OptionDoc("job_timeout_s", "Seconds to wait for an asynchronous job.", + default=300), + OptionDoc("use_lock", "Take the resource-group lock around configuration " + "changes (needed on some models/configurations).", default=False), + OptionDoc("lock_wait_s", "waitTime for the resource-group lock.", default=30), + ) + + def __init__(self, ctx): + super().__init__(ctx) + ports = self.option("ports") + if isinstance(ports, str): + ports = [p for p in ports.replace(",", " ").split() if p] + if not ports or not all(isinstance(p, str) and _PORT_RE.match(p.upper()) + for p in ports): + raise ArrayError(f"Hitachi VSP option 'ports' must list port ids like " + f"'CL1-A'; got {self.option('ports')!r}") + self._ports: list[str] = list(dict.fromkeys(p.upper() for p in ports)) + try: + self._pool = int(self.option("pool_id")) + except (TypeError, ValueError) as e: + raise ArrayError("Hitachi VSP option 'pool_id' must be a number") from e + self._range: tuple[int, int] | None = None + rng = self.option("ldev_range") + if rng: + m = _RANGE_RE.match(str(rng)) + if not m or int(m.group(1)) > int(m.group(2)): + raise ArrayError(f"Hitachi VSP option 'ldev_range' must be 'start-end'; " + f"got {rng!r}") + self._range = (int(m.group(1)), int(m.group(2))) + sdid = self.option("storage_device_id") + self._base = f"{ROOT}/storages/{sdid}" if sdid else ROOT + self._session_id: Any = None + self._port_types: dict[str, Protocol | None] = {} + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.headers.pop("Authorization", None) + self._session_id = None + resp = self.session.post(f"{self._base}/sessions", + auth=(self.secret("username"), self.secret("password"))) + body = json_of(resp) or {} + token = body.get("token") + if not token: + raise ArrayRequestError("Hitachi VSP session request returned no token") + self._session_id = body.get("sessionId") + self.session.headers["Authorization"] = f"Session {token}" + + def close(self) -> None: + if self._session_id is not None and "Authorization" in self.session.headers: + try: + self.session.delete(f"{self._base}/sessions/{self._session_id}") + except Exception: + pass + self.session.headers.pop("Authorization", None) + self._session_id = None + + # -- request helpers ----------------------------------------------------- + + def _req(self, method: str, path: str, *, params: Any = None, + json: Any = None) -> tuple[int, Any]: + full = path if path.startswith(ROOT) else f"{self._base}{path}" + resp = self.session.request(method, full, params=params, json=json, + ok=_OWN_STATUSES) + try: + body = json_of(resp) + except ArrayRequestError: + body = resp.text + if resp.status_code >= 400: + raise _classify(f"{method} {path}", resp.status_code, body) + return resp.status_code, body + + def _get(self, path: str, **params: Any) -> Any: + return self._req("GET", path, params=params or None)[1] + + def _data(self, path: str, **params: Any) -> list[dict[str, Any]]: + return list((self._get(path, **params) or {}).get("data", [])) + + def _run(self, method: str, path: str, json: Any = None) -> list[str]: + """Issue a change request, wait for its job and return the job's + ``affectedResources``.""" + status, body = self._req(method, path, json=json) + if not isinstance(body, dict) or "jobId" not in body: + return [] + job = self._wait(body) + if job.get("state") != "Succeeded": + raise _classify(f"{method} {path} (job {job.get('jobId')} {job.get('state')})", + None, job.get("error") or {"message": "job did not succeed"}) + return [str(r) for r in job.get("affectedResources") or []] + + def _wait(self, job: dict[str, Any]) -> dict[str, Any]: + timeout = float(self.option("job_timeout_s")) + sleep = self.session.sleep + deadline = _monotonic() + timeout + interval, waited = 0.5, 0.0 + job_id = job["jobId"] + while job.get("status") != "Completed": + if waited >= timeout or _monotonic() > deadline: + raise ArrayRequestError( + f"Hitachi VSP job {job_id} not completed after {timeout:g}s " + f"(status {job.get('status')}, state {job.get('state')}); " + "it may still finish on the array") + sleep(interval) + waited += interval + interval = min(interval * 2, 5.0) + job = self._get(f"/jobs/{job_id}") or {} + return job + + @contextmanager + def _change(self) -> Iterator[None]: + """Resource-group lock around a configuration change (``use_lock``).""" + if not self.option("use_lock"): + yield + return + svc = "/services/resource-group-service/actions" + self._run("POST", f"{svc}/lock/invoke", + {"parameters": {"waitTime": int(self.option("lock_wait_s"))}}) + try: + yield + finally: + try: + self._run("POST", f"{svc}/unlock/invoke", {}) + except ArrayError: + pass + + # -- volumes -------------------------------------------------------------- + + @staticmethod + def _defined(ldev: dict[str, Any] | None) -> bool: + return bool(ldev) and (ldev or {}).get("emulationType", NOT_DEFINED) != NOT_DEFINED + + @staticmethod + def _volume(ldev: dict[str, Any]) -> Volume: + naa = str(ldev.get("naaId") or "").lower() + return Volume(id=str(ldev["ldevId"]), name=str(ldev.get("label") or ""), + size_bytes=int(ldev["blockCapacity"]) * BLOCK, + wwn=f"naa.{naa}" if naa else None) + + def _ldev(self, volume_id: str) -> dict[str, Any] | None: + if not str(volume_id).isdigit(): + return None + try: + ldev = self._get(f"/ldevs/{int(volume_id)}") + except ArrayNotFoundError: + return None + return ldev if self._defined(ldev) else None + + def _require(self, volume_id: str) -> dict[str, Any]: + ldev = self._ldev(volume_id) + if ldev is None: + raise ArrayNotFoundError(f"LDEV {volume_id} is not defined") + return ldev + + def get_info(self) -> ArrayInfo: + sdid = self.option("storage_device_id") + info = self._get(f"{ROOT}/storages/{sdid}" if sdid else f"{ROOT}/storages/instance") + info = info or {} + return ArrayInfo(vendor=self.vendor, model=str(info.get("model", "")), + version=str(info.get("dkcMicroVersion", "")), + serial=str(info.get("serialNumber", ""))) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + body: dict[str, Any] = {"poolId": self._pool, "blockCapacity": size_bytes // BLOCK} + if self._range: + body["startLdevId"], body["endLdevId"] = self._range + with self._change(): + affected = self._run("POST", "/ldevs", body) + ldev_ids = [_tail(r) for r in affected if "/ldevs/" in r] + if not ldev_ids: + raise ArrayRequestError(f"Hitachi VSP LDEV create for {name} reported no LDEV") + ldev_id = ldev_ids[0] + try: + self._run("PUT", f"/ldevs/{ldev_id}", {"label": name}) + except ArrayError: + # An unlabeled LDEV cannot be found again; do not leak it. + try: + self._run("DELETE", f"/ldevs/{ldev_id}") + except ArrayError: + pass + raise + return self._volume(self._require(ldev_id)) + + def get_volume(self, volume_id: str) -> Volume | None: + ldev = self._ldev(volume_id) + return self._volume(ldev) if ldev else None + + def find_volume(self, name: str) -> Volume | None: + head, end = self._range if self._range else (0, None) + found: list[dict[str, Any]] = [] + while True: + page = self._data("/ldevs", ldevOption="dpVolume", poolId=self._pool, + headLdevId=head, count=PAGE_SIZE) + for ldev in page: + if end is not None and int(ldev["ldevId"]) > end: + break + if ldev.get("label") == name and self._defined(ldev): + found.append(ldev) + if len(page) < PAGE_SIZE: + break + head = int(page[-1]["ldevId"]) + 1 + if end is not None and head > end: + break + if len(found) > 1: + raise ArrayConflictError( + f"Several LDEVs labeled {name!r}: " + + ", ".join(str(f["ldevId"]) for f in found)) + if not found: + return None + # List entries may omit fields; read the LDEV itself. + return self.get_volume(str(found[0]["ldevId"])) + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + ldev = self._require(volume_id) + add = size_bytes // BLOCK - int(ldev["blockCapacity"]) + if add > 0: + with self._change(): + self._run("POST", f"/ldevs/{ldev['ldevId']}/actions/expand/invoke", + {"parameters": {"additionalBlockCapacity": add}}) + return self._volume(self._require(volume_id)) + + def delete_volume(self, volume_id: str) -> None: + ldev = self._require(volume_id) + with self._change(): + self._run("DELETE", f"/ldevs/{ldev['ldevId']}") + + # -- ports and host groups ------------------------------------------------------ + + def _port_protocol(self, port: str) -> Protocol | None: + if port not in self._port_types: + info = self._get(f"/ports/{port}") or {} + self._port_types[port] = _PORT_TYPES.get(str(info.get("portType", "")).upper()) + return self._port_types[port] + + def _ports_for(self, protocol: Protocol) -> list[str]: + return [p for p in self._ports if self._port_protocol(p) == protocol] + + def _groups(self, port: str) -> list[dict[str, Any]]: + return self._data("/host-groups", portId=port) + + @staticmethod + def _by_name(groups: list[dict[str, Any]], name: str) -> dict[str, Any] | None: + return next((g for g in groups if g.get("hostGroupName") == name), None) + + def _host_groups(self, name: str) -> list[tuple[str, int]]: + """(port, hostGroupNumber) of host ``name`` on every configured port.""" + out = [] + for port in self._ports: + g = self._by_name(self._groups(port), name) + if g is not None: + out.append((port, int(g["hostGroupNumber"]))) + return out + + def _members(self, protocol: Protocol, port: str, number: int) -> set[str]: + if protocol == Protocol.FC: + items = self._data("/host-wwns", portId=port, hostGroupNumber=number) + return {str(i.get("hostWwn", "")).lower() for i in items} + items = self._data("/host-iscsis", portId=port, hostGroupNumber=number) + return {str(i.get("iscsiName", "")).lower() for i in items} + + def _add_member(self, protocol: Protocol, port: str, number: int, address: str) -> None: + if protocol == Protocol.FC: + self._run("POST", "/host-wwns", + {"hostWwn": address, "portId": port, "hostGroupNumber": number}) + else: + self._run("POST", "/host-iscsis", + {"iscsiName": address, "portId": port, "hostGroupNumber": number}) + + def ensure_host(self, spec: HostSpec) -> str: + wanted = {p: spec.addresses(p) for p in (Protocol.FC, Protocol.ISCSI)} + wanted = {p: a for p, a in wanted.items() if a} + plan: list[tuple[Protocol, str, list[dict[str, Any]]]] = [] + for protocol, addresses in wanted.items(): + ports = self._ports_for(protocol) + if not ports: + raise ArrayError(f"Hitachi VSP array {self.config.id} has no configured " + f"{protocol.value} port (option 'ports': {self._ports})") + for port in ports: + groups = self._groups(port) + # Ownership check before any change: FC refuses a WWN in two + # host groups of a port, iSCSI does not, so scan both. + for g in groups: + if g.get("hostGroupName") == spec.name: + continue + taken = self._members(protocol, port, int(g["hostGroupNumber"])) + clash = sorted(set(addresses) & taken) + if clash: + raise ArrayConflictError( + f"Initiator {clash[0]} is registered to host group " + f"{g.get('hostGroupName')!r} ({g.get('hostGroupId')}) " + f"on port {port}") + plan.append((protocol, port, groups)) + with self._change(): + for protocol, port, groups in plan: + current = self._by_name(groups, spec.name) + if current is None: + body: dict[str, Any] = {"portId": port, "hostGroupName": spec.name, + "hostMode": self.option("host_mode")} + if self.option("host_mode_options"): + body["hostModeOptions"] = [int(o) for o in + self.option("host_mode_options")] + affected = self._run("POST", "/host-groups", body) + ids = [_tail(r) for r in affected if "/host-groups/" in r] + if ids: + number = int(ids[0].split(",")[1]) + else: + found = self._by_name(self._groups(port), spec.name) + if found is None: + raise ArrayRequestError( + f"Host group {spec.name} not found on {port} after create") + number = int(found["hostGroupNumber"]) + have: set[str] = set() + else: + number = int(current["hostGroupNumber"]) + have = self._members(protocol, port, number) + for address in wanted[protocol]: + if address.lower() not in have: + self._add_member(protocol, port, number, address) + return spec.name + + def delete_host(self, host_id: str) -> None: + targets = [(p, n) for p, n in self._host_groups(host_id) if n != 0] + if not targets: + return + with self._change(): + for port, number in targets: + try: + self._run("DELETE", f"/host-groups/{port},{number}") + except ArrayNotFoundError: + pass + + # -- mappings -------------------------------------------------------------------- + + def _paths(self, ldev: dict[str, Any], host_id: str | None = None + ) -> list[dict[str, Any]]: + order = {p: i for i, p in enumerate(self._ports)} + paths = [p for p in ldev.get("ports") or [] + if p.get("portId") in order + and (host_id is None or p.get("hostGroupName") == host_id)] + return sorted(paths, key=lambda p: order[p["portId"]]) + + def _mappings(self, ldev: dict[str, Any], host_id: str | None = None) -> list[Mapping]: + out: dict[str, Mapping] = {} + for p in self._paths(ldev, host_id): + name = str(p.get("hostGroupName")) + out.setdefault(name, Mapping(volume_id=str(ldev["ldevId"]), host_id=name, + lun=int(p["lun"]))) + return list(out.values()) + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + ldev = self._require(volume_id) + groups = self._host_groups(host_id) + if not groups: + raise ArrayNotFoundError(f"Host {host_id} has no host group on " + f"ports {', '.join(self._ports)}") + present = {(p["portId"], int(p["hostGroupNumber"])): int(p["lun"]) + for p in self._paths(ldev, host_id)} + missing = [g for g in groups if g not in present] + if missing: + used: set[int] = set() + for port, number in missing: + used |= {int(lu["lun"]) for lu in + self._data("/luns", portId=port, hostGroupNumber=number)} + preferred = next(iter(present.values()), None) + lun = preferred if preferred is not None and preferred not in used else next( + (n for n in range(MAX_LUN + 1) if n not in used), None) + if lun is None: + raise ArrayConflictError(f"Host {host_id} has no free LUN") + with self._change(): + for port, number in missing: + self._run("POST", "/luns", {"portId": port, "hostGroupNumber": number, + "ldevId": int(ldev["ldevId"]), "lun": lun}) + ldev = self._require(volume_id) + mappings = self._mappings(ldev, host_id) + if not mappings: + raise ArrayRequestError(f"LDEV {volume_id} shows no path to {host_id} after mapping") + return mappings[0] + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + ldev = self._ldev(volume_id) + if ldev is None: + return + paths = self._paths(ldev, host_id) + if not paths: + return + with self._change(): + for p in paths: + try: + self._run("DELETE", + f"/luns/{p['portId']},{p['hostGroupNumber']},{p['lun']}") + except ArrayNotFoundError: + pass + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return self._mappings(self._require(volume_id)) diff --git a/simplyblock_core/storage_arrays/adapters/hpe_nimble.py b/simplyblock_core/storage_arrays/adapters/hpe_nimble.py new file mode 100644 index 0000000000..35cd1583ea --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/hpe_nimble.py @@ -0,0 +1,321 @@ +"""HPE Alletra 5000/6000 and Nimble Storage, NimbleOS REST API v1. + +The endpoint normally includes the API port: ``https://array:5392``. + +Auth: ``POST /v1/tokens {"data": {"username", "password"}}`` returns +``data.session_token`` (sent as ``X-Auth-Token``) and ``data.id`` (used to +release the token with ``DELETE /v1/tokens/``). Every request and +response body is wrapped in ``{"data": ...}``; errors are +``{"messages": [{"code": "SM_...", "severity": "error", "text": "..."}]}``. + +Volumes: sizes are in MiB. ``Volume.id`` is the NimbleOS object id. A volume +must be taken offline (``PUT {"data": {"online": false}}``) before it can be +deleted. + +Hosts are *initiator groups* (``access_protocol`` ``iscsi`` or ``fc``); the +host id is the group id. Initiators are added one by one with ``POST +/v1/initiators``; FC WWPNs use the colon form. Before adding, the adapter +looks each initiator up and raises :class:`ArrayConflictError` if another +group owns it (whether NimbleOS itself forbids an initiator in two groups is +not relied on). Mappings are *access control records* (ACRs) with +``apply_to: "both"`` (volume and its snapshots); the array assigns the LUN. + +Volume identity: NimbleOS volumes report a 32-hex ``serial_number`` and +present it to hosts as a 16-byte **EUI-64** designator (multipath WWID +``2``, sysfs ``wwid`` ``eui.``), not an NAA. The adapter +reports it as ``wwn="eui."`` (``Volume.wwn`` holds the SCSI +designator, NAA or EUI-64), so the identifier equals the host's sysfs wwid. + +Verify against a real system +---------------------------- +* The EUI-64 derivation above (``eui.`` + ``serial_number`` lowercase), both + for iSCSI and FC, on NimbleOS 5.x/6.x. +* Error codes: ``SM_http_unauthorized`` (401) for bad login and expired + tokens; ``SM_eexist`` (409) for duplicate names and ACRs; + ``SM_http_not_found``/``SM_enoent`` (404) for unknown ids; ``SM_vol_online`` + (409, deleting an online volume); ``SM_vol_has_acl`` / ACR-related refusal + when offlining a connected volume; ``SM_invalid_arg`` (400). The adapter + keys on HTTP status, and treats 400 + ``SM_enoent`` / "not found" as + not-found. +* Whether ``GET /v1/volumes/`` for a malformed id answers 400 or 404 + (both are treated as "missing" by ``get_volume``). +* Query filtering on any attribute (``/v1/initiators/detail?iqn=...``, + ``/v1/access_control_records/detail?vol_id=...``) and that a filtered list + with no hits returns ``data: []`` rather than 404. +* LUN assignment: with volume-scoped iSCSI targets (the default on recent + NimbleOS) every iSCSI ACR shows ``lun`` 0; group-scoped targets and FC get + distinct LUNs per initiator group. +* That ``label`` is required for iSCSI initiators (max 64 chars) and is + unique only within its group. +* Shrinking (``PUT size`` below current) is refused; we never request it. +""" +from __future__ import annotations + +import hashlib +from typing import Any +from urllib.parse import quote + +import requests + +from ..base import ( + ArrayAdapter, + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, + colon_wwpn, + normalize_wwpn, +) +from ..http import body_text, json_of + +MIB = 1 << 20 +_NOT_FOUND_CODES = frozenset({"SM_enoent", "SM_http_not_found", "SM_no_such_object"}) + + +def nimble_eui(serial: str) -> str: + return "eui." + serial.lower() + + +class HpeNimbleAdapter(ArrayAdapter): + vendor = "hpe-nimble" + display_name = "HPE Alletra 5000/6000 / Nimble" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC}), + size_granularity_bytes=MIB, + min_size_bytes=MIB, + max_name_len=63, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("pool", "Pool name new volumes are created in (default: the " + "array's default pool)."), + OptionDoc("performance_policy", "Performance policy name for new volumes " + "(default: the array default)."), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._token_id: str | None = None + self._create_extra: dict[str, str] | None = None + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.headers.pop("X-Auth-Token", None) + data = self._data("POST", "/v1/tokens", + json={"data": {"username": self.secret("username"), + "password": self.secret("password")}}) + token = (data or {}).get("session_token") + if not token: + raise ArrayRequestError("NimbleOS login returned no session_token") + self._token_id = data.get("id") + self.session.headers["X-Auth-Token"] = token + + def close(self) -> None: + token_id, self._token_id = self._token_id, None + if token_id and "X-Auth-Token" in self.session.headers: + try: + self.session.delete(f"/v1/tokens/{quote(token_id, safe='')}", + ok=(401, 404)) + except Exception: + pass + self.session.headers.pop("X-Auth-Token", None) + + # -- request helpers ----------------------------------------------------- + + def _request(self, method: str, path: str, *, params: Any = None, json: Any = None, + missing_ok: bool = False) -> requests.Response | None: + resp = self.session.request(method, path, params=params, json=json, + ok=(400, 401, 403, 404, 409)) + status = resp.status_code + if status < 400: + return resp + codes: list[str] = [] + text = body_text(resp) + try: + messages = (resp.json() or {}).get("messages") or [] + codes = [str(m.get("code", "")) for m in messages] + text = "; ".join(f"{m.get('code', '')}: {m.get('text', '')}" for m in messages) or text + except (ValueError, AttributeError): + pass + msg = f"{method} {path} -> HTTP {status}: {text}" + if status in (401, 403): + raise ArrayAuthError(msg) + if status == 404 or any(c in _NOT_FOUND_CODES for c in codes): + if missing_ok: + return None + raise ArrayNotFoundError(msg) + if status == 409 or "SM_eexist" in codes: + raise ArrayConflictError(msg) + raise ArrayRequestError(msg, status=status) + + def _data(self, method: str, path: str, **kw: Any) -> Any: + resp = self._request(method, path, **kw) + if resp is None: + return None + return (json_of(resp) or {}).get("data") + + def _list(self, path: str, **filters: str) -> list[dict[str, Any]]: + return list(self._data("GET", path, params=filters) or []) + + @staticmethod + def _volume(item: dict[str, Any]) -> Volume: + serial = item.get("serial_number") or "" + return Volume(id=item["id"], name=item["name"], size_bytes=int(item["size"]) * MIB, + wwn=nimble_eui(serial) if serial else None) + + def _lookup_id(self, path: str, name: str, what: str) -> str: + items = self._list(path, name=name) + if not items: + raise ArrayError(f"{self.display_name} array {self.config.id}: " + f"{what} {name!r} not found") + return str(items[0]["id"]) + + def _volume_extra(self) -> dict[str, str]: + if self._create_extra is None: + extra: dict[str, str] = {} + if self.option("pool"): + extra["pool_id"] = self._lookup_id("/v1/pools/detail", self.option("pool"), + "pool") + if self.option("performance_policy"): + extra["perfpolicy_id"] = self._lookup_id( + "/v1/performance_policies/detail", self.option("performance_policy"), + "performance policy") + self._create_extra = extra + return self._create_extra + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + arrays = self._list("/v1/arrays/detail") + arr = arrays[0] if arrays else {} + return ArrayInfo(vendor=self.vendor, model=str(arr.get("model", "")), + version=str(arr.get("version", "")), + serial=str(arr.get("serial", ""))) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + body = {"name": name, "size": -(-size_bytes // MIB), **self._volume_extra()} + return self._volume(self._data("POST", "/v1/volumes", json={"data": body})) + + def get_volume(self, volume_id: str) -> Volume | None: + try: + data = self._data("GET", f"/v1/volumes/{quote(volume_id, safe='')}", + missing_ok=True) + except ArrayRequestError as e: + if e.status == 400: # malformed id: cannot exist + return None + raise + return self._volume(data) if data else None + + def find_volume(self, name: str) -> Volume | None: + items = [i for i in self._list("/v1/volumes/detail", name=name) if i.get("name") == name] + return self._volume(items[0]) if items else None + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + data = self._data("PUT", f"/v1/volumes/{quote(volume_id, safe='')}", + json={"data": {"size": -(-size_bytes // MIB)}}) + return self._volume(data) + + def delete_volume(self, volume_id: str) -> None: + path = f"/v1/volumes/{quote(volume_id, safe='')}" + self._data("PUT", path, json={"data": {"online": False}}) + self._request("DELETE", path) + + # -- hosts (initiator groups) ------------------------------------------- + + def _group(self, name: str) -> dict[str, Any] | None: + items = [g for g in self._list("/v1/initiator_groups/detail", name=name) + if g.get("name") == name] + return items[0] if items else None + + def _owner(self, protocol: str, key: str, value: str) -> str | None: + """Name of the initiator group that owns ``value`` (iqn or wwpn).""" + for i in self._list("/v1/initiators/detail", access_protocol=protocol, **{key: value}): + have = str(i.get(key, "")) + same = (normalize_wwpn(have) == normalize_wwpn(value) if key == "wwpn" + else have.lower() == value.lower()) + if same: + return str(i.get("initiator_group_name") or i.get("initiator_group_id")) + return None + + def ensure_host(self, spec: HostSpec) -> str: + unsupported = [p for p in spec.protocols if not p.is_scsi] + if unsupported: + raise ArrayError(f"{self.display_name} adapter does not support " + + ", ".join(sorted(p.value for p in unsupported))) + iqns = spec.addresses(Protocol.ISCSI) + wwpns = [colon_wwpn(w) for w in spec.addresses(Protocol.FC)] + if iqns and wwpns: + raise ArrayError(f"{self.display_name}: an initiator group is either iSCSI or " + "FC; register the host once per protocol") + protocol, key, wanted = ("iscsi", "iqn", iqns) if iqns else ("fc", "wwpn", wwpns) + + group = self._group(spec.name) + if group is not None and group.get("access_protocol") != protocol: + raise ArrayConflictError( + f"Initiator group {spec.name} exists with access_protocol " + f"{group.get('access_protocol')}, not {protocol}") + if group is not None: + list_key = "iscsi_initiators" if protocol == "iscsi" else "fc_initiators" + have = {str(i.get(key, "")).lower() for i in group.get(list_key) or []} + wanted = [w for w in wanted if w.lower() not in have] + for value in wanted: + owner = self._owner(protocol, key, value) + if owner is not None and owner != spec.name: + raise ArrayConflictError( + f"Initiator {value} already belongs to initiator group {owner}") + if group is None: + group = self._data("POST", "/v1/initiator_groups", + json={"data": {"name": spec.name, "access_protocol": protocol, + "description": "simplyblock"}}) + group_id = str(group["id"]) + for value in wanted: + body: dict[str, Any] = {"initiator_group_id": group_id, + "access_protocol": protocol, key: value} + if protocol == "iscsi": + body["label"] = f"sb-{hashlib.sha1(value.encode()).hexdigest()[:12]}" + self._data("POST", "/v1/initiators", json={"data": body}) + return group_id + + def delete_host(self, host_id: str) -> None: + self._request("DELETE", f"/v1/initiator_groups/{quote(host_id, safe='')}", + missing_ok=True) + + # -- mappings (access control records) ----------------------------------- + + def _acrs(self, volume_id: str, host_id: str | None = None) -> list[dict[str, Any]]: + items = self._list("/v1/access_control_records/detail", vol_id=volume_id) + return [a for a in items if a.get("vol_id") == volume_id and a.get("initiator_group_id") + and (host_id is None or a.get("initiator_group_id") == host_id)] + + @staticmethod + def _mapping(acr: dict[str, Any]) -> Mapping: + lun = acr.get("lun") + return Mapping(volume_id=acr["vol_id"], host_id=acr["initiator_group_id"], + lun=int(lun) if lun is not None else None) + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + existing = self._acrs(volume_id, host_id) + if existing: + return self._mapping(existing[0]) + acr = self._data("POST", "/v1/access_control_records", + json={"data": {"vol_id": volume_id, "initiator_group_id": host_id, + "apply_to": "both"}}) + return self._mapping(acr) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + for acr in self._acrs(volume_id, host_id): + self._request("DELETE", + f"/v1/access_control_records/{quote(str(acr['id']), safe='')}", + missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return [self._mapping(a) for a in self._acrs(volume_id)] diff --git a/simplyblock_core/storage_arrays/adapters/hpe_wsapi.py b/simplyblock_core/storage_arrays/adapters/hpe_wsapi.py new file mode 100644 index 0000000000..9642886ec8 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/hpe_wsapi.py @@ -0,0 +1,317 @@ +"""HPE WSAPI: Alletra Storage MP B10000, Primera and 3PAR StoreServ. + +Auth: ``POST /api/v1/credentials {"user", "password"}`` returns ``{"key"}``, +sent as the ``X-HP3PAR-WSAPI-SessionKey`` header on every later request and +released with ``DELETE /api/v1/credentials/``. An expired or unknown key +is answered with HTTP 401/403 (mapped to :class:`ArrayAuthError`, so the client +logs in again). + +Objects are addressed by name: ``Volume.id`` is the volume name and the host +id is the host name. Errors carry ``{"code": , "desc": "..."}``. + +Volumes: ``POST /api/v1/volumes {name, cpg, sizeMiB, tpvv|reduce}``; grow is +``PUT /api/v1/volumes/ {"action": 3, "sizeMiB": }`` -- GROW +takes the *delta*, not the new size. SCSI identity: ``naa.`` from the +volume's ``wwn`` field (``60002AC...``). + +Hosts: ``POST /api/v1/hosts {name, persona, iSCSINames|FCWWNs}``; paths are +added with ``PUT /api/v1/hosts/ {"pathOperation": 1, ...}``. Note the +asymmetry: requests use ``iSCSINames``/``FCWWNs``, responses report +``iSCSIPaths[].name``/``FCPaths[].wwn``. + +VLUNs (mappings): ``POST /api/v1/vluns {volumeName, hostname, lun, autoLun, +maxAutoLun}`` answers 201 with an empty body and a ``Location`` header +``/api/v1/vluns/,,``. ``GET /api/v1/vluns?query="volumeName EQ +"`` lists both the VLUN *template* and one *active* VLUN per host path, +so entries are de-duplicated by (volume, host). + +Only iSCSI and FC hosts are supported: NVMe/TCP and NVMe/FC hosts exist on +Alletra MP and Primera but their WSAPI host fields (``NVMETCPPaths``/NQN +attributes) are not verified, so NVMe is refused up front. + +Verify against a real system +---------------------------- +* Error codes (taken from python-3parclient / Cinder usage, not from a + spec): 23 NON_EXISTENT_VOL (404), 22 EXISTENT_VOL (409), 17 + NON_EXISTENT_HOST (404), 16 EXISTENT_HOST (409), 73 EXISTENT_PATH -- WWN/IQN + already owned by another host (409), 19 NON_EXISTENT_VLUN (404), 29 + EXISTENT_VLUN (409), 34 VOLUME_HAS_VLUNS / 26 HOST_IN_USE (409), 5 invalid + login (403), 6 invalid/expired session key (401). The adapter keys its + behaviour on HTTP status first and on codes only where listed in + ``_NOT_FOUND_CODES`` / ``_CONFLICT_CODES``. +* Whether a 403 is ever returned for a non-auth reason (it would trigger a + pointless re-login before failing). +* Size granularity: 3PAR/Primera allocate in 256 MiB units and report the + rounded ``sizeMiB``; we therefore round to 256 MiB (min 256 MiB) so the + requested and reported sizes agree. Alletra MP may accept 1 MiB units. +* ``reduce: true`` (dedup + compression) as the Alletra MP create flag, versus + ``tdvv``/``compression`` on Primera; ``tpvv: true`` for thin on 3PAR/Primera. +* Whether one host may carry both iSCSI and FC paths (passed through as given). +* The exact ``query`` syntax (``"volumeName EQ x"``, double quotes included) + and that it returns ``{"total", "members"}`` with an empty list (not 404) + when nothing matches; whether active VLUN entries carry ``active: true``. +* Persona 2 = "Generic-ALUA" (persona 1 = Generic, 11 = VMware). +* Max session count per user (stale keys from crashed processes may exhaust + it); we always ``DELETE`` the key on close. +""" +from __future__ import annotations + +import re +from typing import Any +from urllib.parse import quote, unquote + +import requests + +from ..base import ( + ArrayAdapter, + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import body_text, json_of + +MIB = 1 << 20 +_GRANULARITY = 256 * MIB + +SESSION_HEADER = "X-HP3PAR-WSAPI-SessionKey" +_API = "/api/v1" + +GROW_VOLUME = 3 +PATH_ADD = 1 + +_NOT_FOUND_CODES = frozenset({17, 19, 23}) +_CONFLICT_CODES = frozenset({16, 22, 26, 29, 34, 73}) + +_LOCATION_RE = re.compile(r"/api/v1/vluns/(?P[^,]+),(?P\d+),(?P[^,/]*)") + + +def wsapi_wwn(wwn: str) -> str: + return "naa." + wwn.lower() + + +class HpeWsapiAdapter(ArrayAdapter): + vendor = "hpe-wsapi" + display_name = "HPE Alletra MP / Primera / 3PAR" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC}), + size_granularity_bytes=_GRANULARITY, + min_size_bytes=_GRANULARITY, + max_name_len=31, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("cpg", "Common Provisioning Group new volumes are created in.", + required=True), + OptionDoc("provisioning", "'tpvv' (thin, 3PAR/Primera) or 'reduce' " + "(thin + dedup + compression, Alletra MP).", default="tpvv"), + OptionDoc("persona", "Host persona id for new hosts (2 = Generic-ALUA).", + default=2), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._key: str | None = None + provisioning = str(self.option("provisioning")).lower() + if provisioning not in ("tpvv", "reduce"): + raise ArrayError(f"{self.display_name}: provisioning must be 'tpvv' or " + f"'reduce', not {provisioning!r}") + self._provisioning = provisioning + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.headers.pop(SESSION_HEADER, None) + resp = self._call("POST", "/credentials", + json={"user": self.secret("username"), + "password": self.secret("password")}) + key = (json_of(resp) or {}).get("key") + if not key: + raise ArrayRequestError("WSAPI login returned no session key") + self._key = key + self.session.headers[SESSION_HEADER] = key + + def close(self) -> None: + key, self._key = self._key, None + self.session.headers.pop(SESSION_HEADER, None) + if key: + try: + self.session.delete(f"{_API}/credentials/{quote(key, safe='')}", + headers={SESSION_HEADER: key}, ok=(401, 403, 404)) + except Exception: + pass + + # -- request helpers ----------------------------------------------------- + + def _call(self, method: str, path: str, *, params: Any = None, json: Any = None, + missing_ok: bool = False) -> requests.Response: + """One WSAPI request; maps ``{"code", "desc"}`` errors to ArrayErrors. + With ``missing_ok`` a not-found answer is returned instead of raised.""" + resp = self.session.request(method, f"{_API}{path}", params=params, json=json, + ok=(400, 401, 403, 404, 409)) + status = resp.status_code + if status < 400: + return resp + code: int | None = None + desc = body_text(resp) + try: + body = resp.json() + if isinstance(body, dict): + code = int(body["code"]) if body.get("code") is not None else None + desc = str(body.get("desc", desc)) + except (ValueError, TypeError, KeyError): + pass + msg = f"{method} {path} -> HTTP {status}" + (f" code {code}" if code is not None else "") \ + + f": {desc}" + if status in (401, 403): + raise ArrayAuthError(msg) + if status == 404 or code in _NOT_FOUND_CODES: + if missing_ok: + return resp + raise ArrayNotFoundError(msg) + if status == 409 or code in _CONFLICT_CODES: + raise ArrayConflictError(msg) + raise ArrayRequestError(msg, status=status) + + @staticmethod + def _found(resp: requests.Response) -> bool: + return resp.status_code < 400 + + @staticmethod + def _volume(item: dict[str, Any]) -> Volume: + wwn = item.get("wwn") or "" + return Volume(id=item["name"], name=item["name"], + size_bytes=int(item["sizeMiB"]) * MIB, + wwn=wsapi_wwn(wwn) if wwn else None) + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + system = json_of(self._call("GET", "/system")) or {} + return ArrayInfo(vendor=self.vendor, model=str(system.get("model", "")), + version=str(system.get("systemVersion", "")), + serial=str(system.get("serialNumber", ""))) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + body: dict[str, Any] = {"name": name, "cpg": self.option("cpg"), + "sizeMiB": -(-size_bytes // MIB)} + body["reduce" if self._provisioning == "reduce" else "tpvv"] = True + self._call("POST", "/volumes", json=body) + vol = self.get_volume(name) + if vol is None: + raise ArrayRequestError(f"Volume {name} not found right after creation") + return vol + + def get_volume(self, volume_id: str) -> Volume | None: + resp = self._call("GET", f"/volumes/{quote(volume_id, safe='')}", missing_ok=True) + return self._volume(json_of(resp)) if self._found(resp) else None + + def find_volume(self, name: str) -> Volume | None: + return self.get_volume(name) + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + vol = self.get_volume(volume_id) + if vol is None: + raise ArrayNotFoundError(f"Volume {volume_id} does not exist") + delta = -(-size_bytes // MIB) - vol.size_bytes // MIB + if delta > 0: + self._call("PUT", f"/volumes/{quote(volume_id, safe='')}", + json={"action": GROW_VOLUME, "sizeMiB": delta}) + elif delta < 0: + raise ArrayConflictError(f"WSAPI cannot shrink volume {volume_id}") + new = self.get_volume(volume_id) + if new is None: + raise ArrayNotFoundError(f"Volume {volume_id} vanished during resize") + return new + + def delete_volume(self, volume_id: str) -> None: + self._call("DELETE", f"/volumes/{quote(volume_id, safe='')}") + + def _host(self, name: str) -> dict[str, Any] | None: + resp = self._call("GET", f"/hosts/{quote(name, safe='')}", missing_ok=True) + return json_of(resp) if self._found(resp) else None + + def ensure_host(self, spec: HostSpec) -> str: + unsupported = [p for p in spec.protocols if not p.is_scsi] + if unsupported: + raise ArrayError(f"{self.display_name} adapter does not support " + + ", ".join(sorted(p.value for p in unsupported))) + iqns = spec.addresses(Protocol.ISCSI) + wwns = [w.upper() for w in spec.addresses(Protocol.FC)] + host = self._host(spec.name) + if host is None: + # Create with one path type; any second type is added below + # (whether WSAPI accepts both in one request is unverified). + body: dict[str, Any] = {"name": spec.name, "persona": int(self.option("persona"))} + if iqns: + body["iSCSINames"] = iqns + else: + body["FCWWNs"] = wwns + self._call("POST", "/hosts", json=body) + if not (iqns and wwns): + return spec.name + host = {"iSCSIPaths": [{"name": i} for i in iqns], "FCPaths": []} + have_iqns = {str(p.get("name", "")).lower() for p in host.get("iSCSIPaths") or []} + have_wwns = {str(p.get("wwn", "")).upper() for p in host.get("FCPaths") or []} + add_iqns = [i for i in iqns if i.lower() not in have_iqns] + add_wwns = [w for w in wwns if w not in have_wwns] + # One path type per request (whether WSAPI accepts both in one + # modify is unverified; separate requests are always valid). + if add_iqns: + self._call("PUT", f"/hosts/{quote(spec.name, safe='')}", + json={"pathOperation": PATH_ADD, "iSCSINames": add_iqns}) + if add_wwns: + self._call("PUT", f"/hosts/{quote(spec.name, safe='')}", + json={"pathOperation": PATH_ADD, "FCWWNs": add_wwns}) + return spec.name + + def delete_host(self, host_id: str) -> None: + self._call("DELETE", f"/hosts/{quote(host_id, safe='')}", missing_ok=True) + + def _vluns(self, volume_id: str, host_id: str | None = None) -> list[Mapping]: + resp = self._call("GET", "/vluns", params={"query": f'"volumeName EQ {volume_id}"'}, + missing_ok=True) + if not self._found(resp): + return [] + seen: dict[tuple[str, str], Mapping] = {} + for m in (json_of(resp) or {}).get("members", []): + host = m.get("hostname") + if not host or m.get("volumeName") != volume_id: + continue + if host_id is not None and host != host_id: + continue + # The template and every active per-path VLUN share (vol, lun, host). + seen.setdefault((volume_id, host), + Mapping(volume_id=volume_id, host_id=host, lun=int(m["lun"]))) + return list(seen.values()) + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + existing = self._vluns(volume_id, host_id) + if existing: + return existing[0] + resp = self._call("POST", "/vluns", + json={"volumeName": volume_id, "hostname": host_id, "lun": 0, + "autoLun": True, "maxAutoLun": 0}) + m = _LOCATION_RE.search(unquote(resp.headers.get("Location", ""))) + if m: + return Mapping(volume_id=volume_id, host_id=host_id, lun=int(m["lun"])) + existing = self._vluns(volume_id, host_id) + if not existing: + raise ArrayRequestError(f"VLUN {volume_id} -> {host_id} not found after creation") + return existing[0] + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + for m in self._vluns(volume_id, host_id): + self._call("DELETE", "/vluns/" + quote(f"{volume_id},{m.lun},{host_id}", safe=","), + missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return self._vluns(volume_id) diff --git a/simplyblock_core/storage_arrays/adapters/huawei_dorado.py b/simplyblock_core/storage_arrays/adapters/huawei_dorado.py new file mode 100644 index 0000000000..9c97615739 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/huawei_dorado.py @@ -0,0 +1,426 @@ +"""Huawei OceanStor Dorado (V6 / OceanStor 6.x) DeviceManager REST API. + +Auth: ``POST /deviceManager/rest/xx/sessions`` with ``{username, password, +scope: "0"}`` (scope 0 = local user) returns ``data.deviceid`` and +``data.iBaseToken`` (plus a session cookie, kept by the ``requests`` +session). Every later call goes to ``/deviceManager/rest//...`` +with the ``iBaseToken`` header. Logout is ``DELETE .../sessions``. + +Envelope: answers are HTTP 200 even on failure, with ``{"data": ..., +"error": {"code": , "description": "..."}}``; code 0 is success. A +query that matches nothing omits ``data``. All values are strings. Codes +mapped here (from Huawei's own CSI driver, eSDK_K8S_Plugin): + +* auth: -401 (not logged in / session expired), 1077949069 (user offline), + 1077987870 / 1077949081 / 1077949061 (bad user or password), + 1077949070 / 1077987871 (account locked), 1077949071 (IP locked) +* not found: 1077948996 (object), 1077936859 (LUN), 1077937498 (host), + 1077937500 (host group), 1077951819 (mapping view) +* conflict: 1077948993 (name exists), 1077948997 (id not unique) +* tolerated as "already done": 1077936862 (LUN already in LUN group), + 1077937501 (host already in host group), 1073745412 (host not in host + group), 1073804556 / 1073804560 (host/LUN group already in mapping view), + 1073804552 / 1073804554 (host/LUN group not in mapping view) + +LUNs: ``POST /lun {NAME, PARENTID: , CAPACITY: <512-byte sectors>, +ALLOCTYPE: 1}`` (thin; Dorado LUNs are always thin), ``PUT /lun/expand {ID, +CAPACITY}``, ``DELETE /lun/``. Identity: ``WWN`` -> ``naa.``. +The ``storage_pool`` option is a pool *name*, resolved once per session via +``GET /storagepool?filter=NAME::``. + +Hosts and initiators: ``POST /host {NAME, OPERATIONSYSTEM: 0}`` (Linux). +iSCSI/FC initiators are objects of their own (``/iscsi_initiator/``, +``/fc_initiator/``) with ``ISFREE`` / ``PARENTID``; one is created if +the array does not know it yet, then bound with ``PUT ... {PARENTTYPE: 21, +PARENTID: }``. An initiator whose ``ISFREE`` is ``"false"`` and +``PARENTID`` is another host raises :class:`ArrayConflictError`. Hosts are +addressed by id (``ensure_host`` returns it). + +Mapping: the classic host group + LUN group + mapping view triple, one per +host, as Huawei's CSI driver does: ``sb_hg_`` holds the host, +``sb_lg_`` holds the LUNs mapped to it, ``sb_mv_`` joins them. +Map/unmap add/remove the LUN to/from the host's LUN group; ``delete_host`` +tears the triple down, unbinds the initiators and deletes the host. The +host LUN id comes from ``GET /lun/associate?TYPE=11&ASSOCIATEOBJTYPE=21& +ASSOCIATEOBJID=`` (``HostLUNID`` inside the ``ASSOCIATEMETADATA`` JSON +string). ``list_mappings`` reports only mappings made through ``sb_lg_`` +groups; a LUN in a customer's LUN group is never touched (deleting it then +fails on the array). + +NVMe hosts are not supported by this adapter (iSCSI and FC only). + +Verify against a real system +---------------------------- +* Size granularity: CAPACITY is in 512-byte sectors, but whether Dorado + accepts any sector count is unverified; we round sizes to 1 MiB to be safe. + Whether CAPACITY stays in 512-byte units for 4K-sector LUNs (``SECTORSIZE``). +* Name limits: LUN names and host names assumed <= 31 characters. +* ``filter=NAME::x`` is assumed to be an exact match (results are + re-filtered by exact name anyway); query values are sent URL-encoded. +* Creating an FC initiator object for a WWPN that has never logged in + (``POST /fc_initiator {ID, TYPE: "223"}``). +* Unbinding initiators: ``PUT /iscsi_initiator/remove_iscsi_from_host`` and + ``PUT /fc_initiator/remove_fc_from_host`` with ``{ID, TYPE}`` (from the + OpenStack Cinder Huawei driver), and listing them with ``?PARENTID=``. +* ``DELETE`` requests carry a JSON body (``/lungroup/associate``, + ``/host/associate``) as the CSI driver sends them. +* Error codes for: deleting a LUN that is still in a LUN group, deleting a + host that still has initiators or group membership, binding an initiator + that is bound elsewhere, and a host already in another host group (a host + can belong to one host group only; the adapter checks this itself and + raises :class:`ArrayConflictError`). +* ``GET /system/`` fields used for ``get_info``: ``ID`` (serial), + ``productModeString`` or ``PRODUCTMODE`` (model code), ``pointRelease`` or + ``PRODUCTVERSION``. +* Some firmware may answer an expired session with HTTP 401 instead of the + -401 envelope; both are treated as an auth error. +""" +from __future__ import annotations + +import json +from typing import Any + +from ..base import ( + ArrayAdapter, + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import json_of + +SECTOR = 512 +REST = "/deviceManager/rest" + +AUTH_CODES = frozenset({-401, 1077949069, 1077987870, 1077949081, 1077949061, + 1077949070, 1077987871, 1077949071}) +NOT_FOUND_CODES = frozenset({1077948996, 1077936859, 1077937498, 1077937500, 1077951819}) +CONFLICT_CODES = frozenset({1077948993, 1077948997}) + +LUN_ALREADY_IN_GROUP = 1077936862 +HOST_ALREADY_IN_GROUP = 1077937501 +HOST_NOT_IN_GROUP = 1073745412 +GROUP_ALREADY_IN_VIEW = frozenset({1073804556, 1073804560}) +GROUP_NOT_IN_VIEW = frozenset({1073804552, 1073804554}) + +OBJ_HOST = 21 +OBJ_HOSTGROUP = 14 +OBJ_LUN = 11 +OBJ_LUNGROUP = 256 +OBJ_MAPPINGVIEW = 245 +TYPE_ISCSI_INITIATOR = "222" +TYPE_FC_INITIATOR = "223" + +_LG, _HG, _MV = "sb_lg_", "sb_hg_", "sb_mv_" + + +def huawei_wwn(wwn: str) -> str: + return "naa." + wwn.lower() + + +class HuaweiDoradoAdapter(ArrayAdapter): + vendor = "huawei-dorado" + display_name = "Huawei OceanStor Dorado" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC}), + size_granularity_bytes=1 << 20, + min_size_bytes=1 << 20, + max_name_len=31, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("storage_pool", "Name of the storage pool new LUNs are created in.", + required=True), + OptionDoc("scope", "Login scope: '0' local user, '1' LDAP user.", default="0"), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._base = "" + self._pool_id: str | None = None + + # -- session ----------------------------------------------------------------- + + def login(self) -> None: + self.session.headers.pop("iBaseToken", None) + self._base = "" + resp = self.session.post(f"{REST}/xx/sessions", json={ + "username": self.secret("username"), "password": self.secret("password"), + "scope": str(self.option("scope"))}) + data = self._unwrap("POST", "/xx/sessions", resp) + if not isinstance(data, dict) or not data.get("deviceid") or not data.get("iBaseToken"): + raise ArrayRequestError("OceanStor login returned no deviceid/iBaseToken") + self.session.headers["iBaseToken"] = data["iBaseToken"] + self._base = f"{REST}/{data['deviceid']}" + + def close(self) -> None: + if self._base and "iBaseToken" in self.session.headers: + try: + self.session.delete(f"{self._base}/sessions") + except Exception: + pass + + # -- request helpers ------------------------------------------------------------ + + @staticmethod + def _unwrap(method: str, path: str, resp, tolerate: frozenset[int] = frozenset(), + missing_ok: bool = False) -> Any: + body = json_of(resp) or {} + err = body.get("error") or {} + try: + code = int(err.get("code", 0)) + except (TypeError, ValueError): + code = -1 + if code == 0 or code in tolerate: + return body.get("data") + msg = f"{method} {path}: error {code}: {err.get('description', '')}".rstrip() + if err.get("suggestion"): + msg += f" ({err['suggestion']})" + if code in AUTH_CODES: + raise ArrayAuthError(msg) + if code in NOT_FOUND_CODES: + if missing_ok: + return None + raise ArrayNotFoundError(msg) + if code in CONFLICT_CODES: + raise ArrayConflictError(msg) + raise ArrayRequestError(msg) + + def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, + body: Any = None, tolerate: frozenset[int] = frozenset(), + missing_ok: bool = False) -> Any: + if not self._base: + raise ArrayAuthError("OceanStor session not established") + resp = self.session.request(method, f"{self._base}{path}", params=params, json=body) + return self._unwrap(method, path, resp, tolerate, missing_ok) + + def _list(self, path: str, params: dict[str, Any] | None = None) -> list[dict[str, Any]]: + data = self._call("GET", path, params=params) + return [d for d in data if isinstance(d, dict)] if isinstance(data, list) else [] + + def _by_name(self, path: str, name: str) -> dict[str, Any] | None: + rows = self._list(path, {"filter": f"NAME::{name}", "range": "[0-100]"}) + return next((r for r in rows if r.get("NAME") == name), None) + + # -- info / pool ------------------------------------------------------------------ + + def get_info(self) -> ArrayInfo: + s = self._call("GET", "/system/") or {} + return ArrayInfo( + vendor=self.vendor, + model=str(s.get("productModeString") or s.get("PRODUCTMODE") or ""), + version=str(s.get("pointRelease") or s.get("PRODUCTVERSION") or ""), + serial=str(s.get("ID", ""))) + + def _pool(self) -> str: + if self._pool_id is None: + name = str(self.option("storage_pool")) + pool = self._by_name("/storagepool", name) + if pool is None: + raise ArrayError(f"OceanStor storage pool {name!r} not found") + self._pool_id = str(pool["ID"]) + return self._pool_id + + # -- LUNs ----------------------------------------------------------------------------- + + @staticmethod + def _volume(lun: dict[str, Any]) -> Volume: + wwn = str(lun.get("WWN") or "") + return Volume(id=str(lun["ID"]), name=lun["NAME"], + size_bytes=int(lun["CAPACITY"]) * SECTOR, + wwn=huawei_wwn(wwn) if wwn else None) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + lun = self._call("POST", "/lun", body={ + "NAME": name, "PARENTID": self._pool(), "CAPACITY": size_bytes // SECTOR, + "ALLOCTYPE": 1, "DESCRIPTION": "simplyblock"}) + return self._volume(lun) + + def get_volume(self, volume_id: str) -> Volume | None: + lun = self._call("GET", f"/lun/{volume_id}", missing_ok=True) + return self._volume(lun) if lun else None + + def find_volume(self, name: str) -> Volume | None: + lun = self._by_name("/lun", name) + return self._volume(lun) if lun else None + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + self._call("PUT", "/lun/expand", body={"ID": volume_id, + "CAPACITY": size_bytes // SECTOR}) + vol = self.get_volume(volume_id) + if vol is None: + raise ArrayNotFoundError(f"LUN {volume_id} does not exist") + return vol + + def delete_volume(self, volume_id: str) -> None: + self._call("DELETE", f"/lun/{volume_id}") + + # -- hosts and initiators --------------------------------------------------------------- + + @staticmethod + def _initiator_kinds(spec: HostSpec) -> list[tuple[str, str, str]]: + """(collection, TYPE, address) for each initiator.""" + out = [("iscsi_initiator", TYPE_ISCSI_INITIATOR, a) + for a in spec.addresses(Protocol.ISCSI)] + out += [("fc_initiator", TYPE_FC_INITIATOR, a) for a in spec.addresses(Protocol.FC)] + return out + + def _initiator(self, collection: str, address: str) -> dict[str, Any] | None: + data = self._call("GET", f"/{collection}/{address}", missing_ok=True) + return data if isinstance(data, dict) and data else None + + def ensure_host(self, spec: HostSpec) -> str: + host = self._by_name("/host", spec.name) + host_id = str(host["ID"]) if host else None + # Check ownership before creating anything. + pending: list[tuple[str, str, str, bool]] = [] # collection, type, address, exists + for collection, itype, address in self._initiator_kinds(spec): + ini = self._initiator(collection, address) + if ini is not None and str(ini.get("ISFREE", "true")).lower() == "false": + parent = str(ini.get("PARENTID") or "") + if parent and parent != host_id: + raise ArrayConflictError( + f"Initiator {address} is registered to another host " + f"({ini.get('PARENTNAME') or parent})") + continue # already ours + pending.append((collection, itype, address, ini is not None)) + if host_id is None: + created = self._call("POST", "/host", body={ + "NAME": spec.name, "TYPE": OBJ_HOST, "OPERATIONSYSTEM": 0, + "DESCRIPTION": "simplyblock"}) + host_id = str(created["ID"]) + for collection, itype, address, exists in pending: + if not exists: + self._call("POST", f"/{collection}", body={"ID": address, "TYPE": itype}, + tolerate=frozenset({1077948997})) + self._call("PUT", f"/{collection}/{address}", + body={"TYPE": itype, "ID": address, + "PARENTTYPE": OBJ_HOST, "PARENTID": host_id}) + return host_id + + def delete_host(self, host_id: str) -> None: + host = self._call("GET", f"/host/{host_id}", missing_ok=True) + if not host: + return + if self._host_luns(host_id): + raise ArrayConflictError(f"Host {host_id} still has LUNs mapped") + view = self._by_name("/mappingview", _MV + host_id) + hg = self._by_name("/hostgroup", _HG + host_id) + lg = self._by_name("/lungroup", _LG + host_id) + if view is not None: + for obj_type, group in ((OBJ_HOSTGROUP, hg), (OBJ_LUNGROUP, lg)): + if group is not None: + self._call("PUT", "/mappingview/remove_associate", body={ + "ID": view["ID"], "ASSOCIATEOBJTYPE": obj_type, + "ASSOCIATEOBJID": group["ID"]}, tolerate=GROUP_NOT_IN_VIEW) + self._call("DELETE", f"/mappingview/{view['ID']}", missing_ok=True) + if hg is not None: + self._call("DELETE", "/host/associate", body={ + "ID": hg["ID"], "ASSOCIATEOBJTYPE": OBJ_HOST, "ASSOCIATEOBJID": host_id}, + tolerate=frozenset({HOST_NOT_IN_GROUP})) + self._call("DELETE", f"/hostgroup/{hg['ID']}", missing_ok=True) + if lg is not None: + self._call("DELETE", f"/lungroup/{lg['ID']}", missing_ok=True) + for collection, itype, action in ( + ("iscsi_initiator", TYPE_ISCSI_INITIATOR, "remove_iscsi_from_host"), + ("fc_initiator", TYPE_FC_INITIATOR, "remove_fc_from_host")): + for ini in self._list(f"/{collection}", {"PARENTID": host_id}): + self._call("PUT", f"/{collection}/{action}", + body={"ID": ini["ID"], "TYPE": itype}) + self._call("DELETE", f"/host/{host_id}", missing_ok=True) + + # -- mapping ------------------------------------------------------------------------------ + + def _host_luns(self, host_id: str) -> dict[str, int | None]: + """LUN id -> host LUN id for every LUN the host sees.""" + rows = self._list("/lun/associate", {"TYPE": OBJ_LUN, "ASSOCIATEOBJTYPE": OBJ_HOST, + "ASSOCIATEOBJID": host_id}) + out: dict[str, int | None] = {} + for r in rows: + lun_no: int | None = None + try: + meta = json.loads(r.get("ASSOCIATEMETADATA") or "{}") + if meta.get("HostLUNID") is not None: + lun_no = int(meta["HostLUNID"]) + except (TypeError, ValueError): + pass + out[str(r["ID"])] = lun_no + return out + + def _ensure_group(self, path: str, name: str, extra: dict[str, Any]) -> str: + found = self._by_name(path, name) + if found is None: + try: + found = self._call("POST", path, body={"NAME": name, **extra}) + except ArrayConflictError: # created concurrently + found = self._by_name(path, name) + if not found: + raise ArrayRequestError(f"Could not create {path[1:]} {name}") + return str(found["ID"]) + + def _ensure_view(self, host_id: str) -> str: + """Create (or complete) the host's group triple; returns the LUN group id.""" + if self._call("GET", f"/host/{host_id}", missing_ok=True) is None: + raise ArrayNotFoundError(f"Host {host_id} does not exist") + lg = self._ensure_group("/lungroup", _LG + host_id, {"APPTYPE": 0}) + hg = self._ensure_group("/hostgroup", _HG + host_id, {}) + groups = self._list("/hostgroup/associate", {"ASSOCIATEOBJTYPE": OBJ_HOST, + "ASSOCIATEOBJID": host_id}) + others = [g for g in groups if str(g.get("ID")) != hg] + if others: + raise ArrayConflictError( + f"Host {host_id} already belongs to host group " + f"{others[0].get('NAME') or others[0].get('ID')}; an OceanStor host can be " + "in one host group only") + if not groups: + self._call("POST", "/hostgroup/associate", body={ + "ID": hg, "ASSOCIATEOBJTYPE": OBJ_HOST, "ASSOCIATEOBJID": host_id}, + tolerate=frozenset({HOST_ALREADY_IN_GROUP})) + mv = self._ensure_group("/mappingview", _MV + host_id, {}) + for obj_type, group in ((OBJ_HOSTGROUP, hg), (OBJ_LUNGROUP, lg)): + self._call("PUT", "/mappingview/create_associate", body={ + "ID": mv, "ASSOCIATEOBJTYPE": obj_type, "ASSOCIATEOBJID": group}, + tolerate=GROUP_ALREADY_IN_VIEW) + return lg + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + seen = self._host_luns(host_id) + if volume_id in seen: + return Mapping(volume_id=volume_id, host_id=host_id, lun=seen[volume_id]) + lg = self._ensure_view(host_id) + self._call("POST", "/lungroup/associate", body={ + "ID": lg, "ASSOCIATEOBJTYPE": OBJ_LUN, "ASSOCIATEOBJID": volume_id}, + tolerate=frozenset({LUN_ALREADY_IN_GROUP, 1077948997})) + seen = self._host_luns(host_id) + if volume_id not in seen: + raise ArrayError(f"LUN {volume_id} not visible to host {host_id} after mapping") + return Mapping(volume_id=volume_id, host_id=host_id, lun=seen[volume_id]) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + lg = self._by_name("/lungroup", _LG + host_id) + if lg is None: + return + self._call("DELETE", "/lungroup/associate", body={ + "ID": lg["ID"], "ASSOCIATEOBJTYPE": OBJ_LUN, "ASSOCIATEOBJID": volume_id}, + missing_ok=True) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + groups = self._list("/lungroup/associate", {"ASSOCIATEOBJTYPE": OBJ_LUN, + "ASSOCIATEOBJID": volume_id}) + out = [] + for g in groups: + name = str(g.get("NAME", "")) + if not name.startswith(_LG): + continue + host_id = name[len(_LG):] + out.append(Mapping(volume_id=volume_id, host_id=host_id, + lun=self._host_luns(host_id).get(volume_id))) + return out diff --git a/simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py b/simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py new file mode 100644 index 0000000000..fe18364fb1 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py @@ -0,0 +1,378 @@ +"""IBM FlashSystem / SAN Volume Controller (IBM Storage Virtualize) REST API. + +The Storage Virtualize REST API (port 7443) is a thin wrapper around the CLI: +every command is ``POST /rest/[/]`` with the command's +flags as a JSON object (``-unit b`` -> ``{"unit": "b"}``, ``-thin`` -> +``{"thin": true}``). Positional arguments go into the path. + +Auth: ``POST /rest/auth`` with headers ``X-Auth-Username`` / +``X-Auth-Password`` returns ``{"token": ...}``; every later call carries +``X-Auth-Token``. Tokens expire after an idle period (one hour by default) +and the array then answers 403, which the client turns into a re-login. + +Errors: a failed command answers HTTP 500 with the CLI's message, e.g. +``CMMVC5753E The specified object does not exist or is not a suitable +candidate.`` The ``CMMVCnnnnE`` code is mapped as follows (anything else is +an :class:`ArrayRequestError`): + +* not found: CMMVC5753E, CMMVC5754E, CMMVC5804E, CMMVC5842E +* conflict: CMMVC6035E (name exists), CMMVC6581E (IQN/NQN already assigned + or invalid), CMMVC5867E (WWPN already assigned or invalid), CMMVC5878E + (volume already mapped to that host), CMMVC6071E (mapped to another host, + needs ``force``), CMMVC5840E (volume still mapped), CMMVC5871E (host + still has mappings) + +Objects: volumes are addressed by their numeric id (``Volume.id``), hosts by +name (``ensure_host`` returns the host name). Note that the array reuses the +lowest free id after a deletion, so a stale volume id can address a +different volume later -- callers pass ``expected_identifier`` (the WWN) to +the client, which then refuses to act on the wrong volume. + +Volumes are created with ``mkvolume`` (``-thin`` unless ``thin`` is False); +the virtual capacity has 512-byte granularity (extents only govern the real +allocation). Resizing grows with ``expandvdisksize`` (a delta). Deletion uses +``rmvolume``. Volume identity: ``vdisk_UID`` (32 hex) -> ``naa.``. + +Initiator ownership: ``mkhost``/``addhostport`` fail with CMMVC6581E / +CMMVC5867E when a port already belongs to another host; the adapter first +reads its own host (``lshost ``, ports in the ``nodes`` list) so ports +it already owns are never re-added, and treats those codes as +:class:`ArrayConflictError`. + +Verify against a real system +---------------------------- +* REST root: ``/rest`` (as used by IBM's Ansible collection). Code level + 8.5+ documents ``/rest/v1``; set the ``rest_root`` option if ``/rest`` is + rejected. +* HTTP status on an expired token (assumed 403, 401 handled the same) and on + bad credentials at ``/rest/auth`` (assumed 403). +* Error body format: plain text with the CMMVC message (JSON bodies are also + parsed). Codes CMMVC5804E, CMMVC5840E (rmvolume of a mapped volume; newer + levels may answer with a different code), CMMVC5871E (rmhost with + mappings) and the NQN-already-assigned code (assumed CMMVC6581E as for + IQNs) are from memory of the CLI reference. +* ``mkhost -protocol`` values for NVMe hosts (``tcpnvme``, ``rdmanvme`` on + 8.6+; older levels only knew ``nvme``) -- set ``nvme_protocol_value`` if + the array rejects them. SCSI hosts are created without ``-protocol``. +* Detail views (``lsvdisk ``, ``lshost ``) may come back as a list + of dicts (object, then copies) or one dict; ``lsvdiskhostmap`` with a + single row may come back as a dict instead of a one-element list. Both + shapes are accepted. +* ``mkvolume -unit b`` with a size that is not a 512-byte multiple (we never + send one). +* ``mkhost -force`` for FC WWPNs that are not logged in (we always pass it). +""" +from __future__ import annotations + +import re +from typing import Any +from urllib.parse import quote + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, +) +from ..http import body_text, json_of + +_CODE_RE = re.compile(r"CMMVC\d{4}[EW]") + +NOT_FOUND_CODES = frozenset({"CMMVC5753E", "CMMVC5754E", "CMMVC5804E", "CMMVC5842E"}) +CONFLICT_CODES = frozenset({ + "CMMVC6035E", # object already exists + "CMMVC6581E", # IQN (and, we assume, NQN) already assigned or not valid + "CMMVC5867E", # WWPN already assigned or not valid + "CMMVC5878E", # volume already mapped to this host + "CMMVC6071E", # volume mapped to another host; needs -force + "CMMVC5840E", # volume not deleted: still mapped + "CMMVC5871E", # host has mappings +}) +#: Codes meaning "this port is owned by another host" on mkhost/addhostport. +PORT_TAKEN_CODES = frozenset({"CMMVC6581E", "CMMVC5867E"}) + +_NVME_PROTOCOL = {Protocol.NVME_TCP: "tcpnvme", Protocol.NVME_RDMA: "rdmanvme"} + + +def ibm_wwn(vdisk_uid: str) -> str: + return "naa." + vdisk_uid.lower() + + +def _rows(data: Any) -> list[dict[str, Any]]: + """List views can come back as a bare dict when there is one row.""" + if data is None: + return [] + if isinstance(data, dict): + return [data] + return [d for d in data if isinstance(d, dict)] + + +def _detail(data: Any) -> dict[str, Any]: + """Detail views: a dict, or a list (object first, then e.g. copies).""" + rows = _rows(data) + return rows[0] if rows else {} + + +class IbmFlashSystemAdapter(ArrayAdapter): + vendor = "ibm-flashsystem" + display_name = "IBM FlashSystem / Storage Virtualize" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC, Protocol.NVME_TCP, + Protocol.NVME_RDMA}), + size_granularity_bytes=512, + min_size_bytes=1 << 20, + max_name_len=63, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("pool", "Storage pool (mdisk group) name or id for new volumes.", + required=True), + OptionDoc("thin", "Create thin-provisioned volumes (mkvolume -thin). In a " + "data reduction pool this is a DRP thin volume.", default=True), + OptionDoc("io_group", "I/O group (name or id) that owns new volumes; " + "default: chosen by the array."), + OptionDoc("rest_root", "REST path root; '/rest/v1' on code levels that " + "reject '/rest'.", default="/rest"), + OptionDoc("nvme_protocol_value", "Override the mkhost -protocol value used " + "for NVMe hosts (default: tcpnvme / rdmanvme by transport)."), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._root = "/" + str(self.option("rest_root")).strip("/") + + # -- session -------------------------------------------------------------- + + def login(self) -> None: + self.session.headers.pop("X-Auth-Token", None) + resp = self.session.post(f"{self._root}/auth", headers={ + "X-Auth-Username": self.secret("username"), + "X-Auth-Password": self.secret("password"), + }) + token = (json_of(resp) or {}).get("token") + if not token: + raise ArrayRequestError("Storage Virtualize /auth returned no token") + self.session.headers["X-Auth-Token"] = token + + def close(self) -> None: + # There is no logout command; tokens lapse on their own. + self.session.headers.pop("X-Auth-Token", None) + + # -- request helper --------------------------------------------------------- + + def _cmd(self, command: str, obj: str | None = None, *, missing_ok: bool = False, + **opts: Any) -> Any: + """Run one CLI command. Returns the decoded JSON (``None`` on an empty + body, or when ``missing_ok`` and the object does not exist).""" + path = f"{self._root}/{command}" + if obj is not None: + path += "/" + quote(str(obj), safe="") + resp = self.session.post(path, json=opts, ok=(500,)) + if resp.status_code == 500: + text = body_text(resp) + m = _CODE_RE.search(text) + code = m.group(0) if m else "" + msg = f"{command}{' ' + str(obj) if obj is not None else ''}: {text.strip()}" + if code in NOT_FOUND_CODES: + if missing_ok: + return None + raise ArrayNotFoundError(msg) + if code in CONFLICT_CODES: + raise ArrayConflictError(msg) + raise ArrayRequestError(msg, status=500) + if not resp.content: + return None + try: + return resp.json() + except ValueError as e: + raise ArrayRequestError(f"{command}: expected JSON, got {body_text(resp)}") from e + + # -- volumes ---------------------------------------------------------------- + + @staticmethod + def _volume(row: dict[str, Any]) -> Volume: + uid = str(row.get("vdisk_UID") or "") + return Volume(id=str(row["id"]), name=row["name"], size_bytes=int(row["capacity"]), + wwn=ibm_wwn(uid) if uid else None) + + def get_info(self) -> ArrayInfo: + s = _detail(self._cmd("lssystem")) + return ArrayInfo(vendor=self.vendor, model=str(s.get("product_name", "")), + version=str(s.get("code_level", "")).split(" ")[0], + serial=str(s.get("id", ""))) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + opts: dict[str, Any] = {"name": name, "pool": str(self.option("pool")), + "size": size_bytes, "unit": "b"} + if self.option("thin"): + opts["thin"] = True + if self.option("io_group") not in (None, ""): + opts["iogrp"] = str(self.option("io_group")) + out = self._cmd("mkvolume", **opts) or {} + vid = out.get("id") + vol = self.get_volume(str(vid)) if vid is not None else self.find_volume(name) + if vol is None: + raise ArrayRequestError(f"mkvolume {name} reported success but the volume " + "cannot be read back") + return vol + + def get_volume(self, volume_id: str) -> Volume | None: + row = _detail(self._cmd("lsvdisk", volume_id, missing_ok=True, bytes=True)) + return self._volume(row) if row else None + + def find_volume(self, name: str) -> Volume | None: + rows = _rows(self._cmd("lsvdisk", filtervalue=f"name={name}", bytes=True)) + rows = [r for r in rows if r.get("name") == name] + return self._volume(rows[0]) if rows else None + + def _require(self, volume_id: str) -> Volume: + vol = self.get_volume(volume_id) + if vol is None: + raise ArrayNotFoundError(f"Volume {volume_id} does not exist") + return vol + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + vol = self._require(volume_id) + delta = size_bytes - vol.size_bytes + if delta < 0: + raise ArrayConflictError( + f"Refusing to shrink volume {vol.name} ({vol.size_bytes} -> {size_bytes})") + if delta: + self._cmd("expandvdisksize", vol.id, size=delta, unit="b") + return self._require(vol.id) + + def delete_volume(self, volume_id: str) -> None: + self._cmd("rmvolume", volume_id) + + # -- hosts ------------------------------------------------------------------- + + def _host(self, name: str) -> dict[str, Any] | None: + data = self._cmd("lshost", name, missing_ok=True) + if data is None: + return None + merged: dict[str, Any] = {} + nodes: list[dict[str, Any]] = [] + for row in _rows(data): + nodes.extend(n for n in row.get("nodes", []) if isinstance(n, dict)) + merged.update({k: v for k, v in row.items() if k != "nodes"}) + merged["nodes"] = nodes + return merged + + @staticmethod + def _ports(host: dict[str, Any]) -> dict[str, set[str]]: + ports: dict[str, set[str]] = {"iscsiname": set(), "fcwwpn": set(), "nqn": set()} + for n in host.get("nodes", []): + if n.get("WWPN"): + ports["fcwwpn"].add(str(n["WWPN"]).lower()) + if n.get("iscsi_name"): + ports["iscsiname"].add(str(n["iscsi_name"]).lower()) + if n.get("nqn"): + ports["nqn"].add(str(n["nqn"])) + return ports + + def _nvme_protocol(self, spec: HostSpec) -> str | None: + nvme = {p for p in spec.protocols if p.is_nvme} + if not nvme: + return None + if len(nvme) > 1: + raise ArrayUnsupportedError( + "A Storage Virtualize NVMe host has one transport; register " + f"{', '.join(sorted(p.value for p in nvme))} initiators as separate hosts") + override = self.option("nvme_protocol_value") + return str(override) if override else _NVME_PROTOCOL[next(iter(nvme))] + + def ensure_host(self, spec: HostSpec) -> str: + wanted = { + "iscsiname": spec.addresses(Protocol.ISCSI), + "fcwwpn": [a.upper() for a in spec.addresses(Protocol.FC)], + "nqn": spec.addresses(Protocol.NVME_TCP, Protocol.NVME_RDMA), + } + protocol = self._nvme_protocol(spec) + host = self._host(spec.name) + try: + if host is None: + opts: dict[str, Any] = {"name": spec.name} + if wanted["iscsiname"]: + opts["iscsiname"] = ",".join(wanted["iscsiname"]) + if wanted["fcwwpn"]: + opts["fcwwpn"] = ":".join(wanted["fcwwpn"]) + opts["force"] = True + if wanted["nqn"]: + opts["nqn"] = ",".join(wanted["nqn"]) + if protocol: + opts["protocol"] = protocol + self._cmd("mkhost", **opts) + return spec.name + have = self._ports(host) + for key, values in wanted.items(): + missing = [v for v in values if v.lower() not in have[key] + and v not in have[key]] + if not missing: + continue + sep = ":" if key == "fcwwpn" else "," + opts = {key: sep.join(missing)} + if key == "fcwwpn": + opts["force"] = True + self._cmd("addhostport", spec.name, **opts) + except ArrayConflictError as e: + code = _CODE_RE.search(str(e)) + if code and code.group(0) in PORT_TAKEN_CODES: + raise ArrayConflictError( + f"An initiator of host {spec.name} is registered to another host " + f"(or is invalid): {e}") from e + raise + return spec.name + + def delete_host(self, host_id: str) -> None: + try: + self._cmd("rmhost", host_id) + except ArrayNotFoundError: + pass + + # -- mappings ------------------------------------------------------------------- + + def _maps(self, volume_id: str) -> list[dict[str, Any]]: + data = self._cmd("lsvdiskhostmap", volume_id, missing_ok=True) + if data is None: + raise ArrayNotFoundError(f"Volume {volume_id} does not exist") + return _rows(data) + + @staticmethod + def _mapping(volume_id: str, row: dict[str, Any]) -> Mapping: + scsi = str(row.get("SCSI_id") or "") + return Mapping(volume_id=volume_id, host_id=str(row["host_name"]), + lun=int(scsi) if scsi.isdigit() else None) + + def list_mappings(self, volume_id: str) -> list[Mapping]: + return [self._mapping(volume_id, r) for r in self._maps(volume_id) + if r.get("host_name")] + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + for m in self.list_mappings(volume_id): + if m.host_id == host_id: + return m + try: + # force: allow the volume to be mapped to more than one host. + self._cmd("mkvdiskhostmap", volume_id, host=host_id, force=True) + except ArrayConflictError as e: + if "CMMVC5878E" not in str(e): # raced with another mapper: fine + raise + for m in self.list_mappings(volume_id): + if m.host_id == host_id: + return m + raise ArrayError(f"Mapping of volume {volume_id} to host {host_id} not found " + "after mkvdiskhostmap") + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + if not any(m.host_id == host_id for m in self.list_mappings(volume_id)): + return + self._cmd("rmvdiskhostmap", volume_id, missing_ok=True, host=host_id) diff --git a/simplyblock_core/storage_arrays/adapters/infinidat.py b/simplyblock_core/storage_arrays/adapters/infinidat.py new file mode 100644 index 0000000000..b87c427806 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/infinidat.py @@ -0,0 +1,290 @@ +"""Infinidat InfiniBox, REST API ``/api/rest`` (InfiniBox 7.x). + +Auth: HTTP basic on every request (``username`` / ``password`` credential). +The array also sets a session cookie, which ``requests`` keeps; basic auth +is still sent, so an expired cookie never matters and a rotated password +surfaces as a 401 (``ArrayClient`` then re-resolves the credential). + +Every response is an envelope ``{"result": ..., "error": ..., "metadata": +{...}}``; errors carry ``error.code`` (e.g. ``VOLUME_NOT_FOUND``) and +``error.message``. Lists are paged (``page``, ``page_size`` up to 1000; +``metadata.pages_total``) and filtered with ``field=eq:value``. + +Destructive requests (delete volume/host, unmap, shrink) need +``?approved=true``; without it the array answers **403** +``APPROVAL_REQUIRED``, which must not be mistaken for an auth failure, so +the adapter always sends the flag and maps that code to +:class:`ArrayRequestError`. + +Objects have integer ids, returned here as strings. Volumes are created in +the pool named by the ``pool`` option (resolved to its id once per session), +thin by default. Sizes are bytes; see granularity below. + +Volume identity: ``serial`` is the SCSI NAA-6 identifier without the +leading ``6`` (``742b0f...``, 31 hex digits, IEEE company id 0x742B0F); +``wwn = "naa.6" + serial``. A 32-digit serial already starting with ``6`` is +used as-is. + +Hosts: ``POST /hosts {name}`` then one ``POST /hosts/{id}/ports +{type, address}`` per initiator (``ISCSI`` IQN; ``FC`` WWPN as 16 lowercase +hex digits). A port owned by another host is refused by the array; when that +happens on a host ``ensure_host`` just created, the empty host is deleted +again. Mappings: ``POST /hosts/{id}/luns {volume_id}`` -> ``{lun, ...}``; +``GET /volumes/{id}/luns`` lists them; ``DELETE +/hosts/{id}/luns/volume_id/{vid}?approved=true`` unmaps. Mappings made +through a host *cluster* (``clustered: true``) are not listed by this adapter +(it never creates them), so ``delete_volume`` on such a volume fails. + +Higher-fidelity testing: Infinidat publishes ``infinisim`` (a Python +InfiniBox simulator serving this REST API) and ``infinisdk`` (the official +SDK). Running the contract suite's live tier against infinisim would check +this adapter's API reading; neither is a dependency here. + +Verify against a real system: + +* Size granularity/minimum: the adapter rounds to 1 MiB (minimum 1 MiB) + because InfiniBox may round byte sizes up to its own allocation unit; if + the array rounds to a coarser unit, raise ``size_granularity_bytes`` (a + silent round-up would make later resizes look like shrinks). +* ``serial`` format (with or without the leading ``6``). +* Error codes and statuses: ``VOLUME_NOT_FOUND``/``HOST_NOT_FOUND``/ + ``POOL_NOT_FOUND`` (404?), name conflict (409 ``*_NAME_CONFLICT``?), + port owned by another host (409 ``PORT_ALREADY_BELONGS_TO_HOST``?), + volume still mapped on delete, ``APPROVAL_REQUIRED`` on unmap/delete. +* FC port address format accepted/returned (colons or not) -- both are + normalised on read. +* That LUN numbers start at 1 and ``GET /volumes/{id}/luns`` includes + ``host_id``. +* ``GET /system`` field names ``serial_number``, ``model_name``, ``version``. +* NVMe/TCP (``NVMEOF`` port type?) is not implemented. +* Name limit (adapter declares 63 characters). +""" +from __future__ import annotations + +from typing import Any + +from ..base import ( + ArrayAdapter, + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + Capabilities, + HostSpec, + Mapping, + OptionDoc, + Protocol, + Volume, + normalize_wwpn, +) +from ..http import body_text + +_API = "/api/rest" +_PAGE_SIZE = 1000 +_APPROVED = {"approved": "true"} + + +def infinibox_wwn(serial: str) -> str: + s = serial.strip().lower() + if len(s) == 32 and s.startswith("6"): + return "naa." + s + return "naa.6" + s + + +def _port_key(ptype: str, address: str) -> tuple[str, str]: + ptype = ptype.upper() + if ptype == "FC": + try: + return ptype, normalize_wwpn(address) + except ValueError: + pass + return ptype, address.strip().lower() + + +class InfinidatAdapter(ArrayAdapter): + vendor = "infinidat-infinibox" + display_name = "Infinidat InfiniBox" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC}), + size_granularity_bytes=1 << 20, + min_size_bytes=1 << 20, + max_name_len=63, + ) + option_docs = ( + OptionDoc("pool", "Name of the pool new volumes are created in.", required=True), + OptionDoc("provtype", "Provisioning type of new volumes: THIN or THICK.", + default="THIN"), + ) + + def __init__(self, ctx): + super().__init__(ctx) + if str(self.option("provtype")).upper() not in ("THIN", "THICK"): + raise ArrayError("InfiniBox option provtype must be THIN or THICK") + self._pool_id: int | None = None + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.http.cookies.clear() + self.session.set_basic_auth(self.secret("username"), self.secret("password")) + self._pool_id = None + + # -- request helpers ----------------------------------------------------- + + def _send(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None, missing_ok: bool = False) -> dict[str, Any] | None: + """Return the response envelope, or ``None`` for a missing object when + ``missing_ok``; map envelope errors to :class:`ArrayError` types.""" + resp = self.session.request(method, _API + path, params=params, json=json, + ok=(400, 403, 404, 409, 500)) + try: + body = resp.json() if resp.content else None + except ValueError: + body = None + env: dict[str, Any] = body if isinstance(body, dict) else {} + err = env.get("error") + status = resp.status_code + if status < 300 and not err: + return env + err = err if isinstance(err, dict) else {} + code = str(err.get("code") or "") + msg = str(err.get("message") or "") or body_text(resp) + text = f"{method} {path} -> HTTP {status}: {code} {msg}".strip() + if status == 403 and code == "APPROVAL_REQUIRED": + raise ArrayRequestError(text, status=status) + if status == 403: + raise ArrayAuthError(text) + if status == 404 or code.endswith("NOT_FOUND"): + if missing_ok: + return None + raise ArrayNotFoundError(text) + if status == 409 or "CONFLICT" in code or "ALREADY" in code: + raise ArrayConflictError(text) + raise ArrayRequestError(text, status=status) + + def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None, missing_ok: bool = False) -> Any: + env = self._send(method, path, params=params, json=json, missing_ok=missing_ok) + return None if env is None else env.get("result") + + def _list(self, path: str, **filters: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + page = 1 + while True: + params: dict[str, Any] = {k: f"eq:{v}" for k, v in filters.items()} + params.update(page=page, page_size=_PAGE_SIZE) + env = self._send("GET", path, params=params) or {} + out.extend(env.get("result") or []) + pages = int((env.get("metadata") or {}).get("pages_total") or 1) + if page >= pages: + return out + page += 1 + + @staticmethod + def _volume(v: dict[str, Any]) -> Volume: + serial = str(v.get("serial") or "") + return Volume(id=str(v["id"]), name=v["name"], size_bytes=int(v["size"]), + wwn=infinibox_wwn(serial) if serial else None) + + @staticmethod + def _int_id(object_id: str) -> int | None: + try: + return int(object_id) + except (TypeError, ValueError): + return None + + def _pool(self) -> int: + if self._pool_id is None: + name = self.option("pool") + pools = self._list("/pools", name=name) + if not pools: + raise ArrayNotFoundError(f"InfiniBox pool {name!r} not found") + self._pool_id = int(pools[0]["id"]) + return self._pool_id + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + system = self._call("GET", "/system") or {} + return ArrayInfo(vendor=self.vendor, model=str(system.get("model_name") or "InfiniBox"), + version=str(system.get("version") or ""), + serial=str(system.get("serial_number") or "")) + + def create_volume(self, name: str, size_bytes: int) -> Volume: + result = self._call("POST", "/volumes", json={ + "name": name, "size": size_bytes, "pool_id": self._pool(), + "provtype": str(self.option("provtype")).upper()}) + return self._volume(result) + + def get_volume(self, volume_id: str) -> Volume | None: + if self._int_id(volume_id) is None: + return None + result = self._call("GET", f"/volumes/{volume_id}", missing_ok=True) + return self._volume(result) if result else None + + def find_volume(self, name: str) -> Volume | None: + items = self._list("/volumes", name=name) + return self._volume(items[0]) if items else None + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + result = self._call("PUT", f"/volumes/{volume_id}", json={"size": size_bytes}) + return self._volume(result) + + def delete_volume(self, volume_id: str) -> None: + self._call("DELETE", f"/volumes/{volume_id}", params=_APPROVED) + + # -- hosts ---------------------------------------------------------------- + + def ensure_host(self, spec: HostSpec) -> str: + wanted = ([("ISCSI", a) for a in spec.addresses(Protocol.ISCSI)] + + [("FC", a) for a in spec.addresses(Protocol.FC)]) + hosts = self._list("/hosts", name=spec.name) + created = not hosts + host = hosts[0] if hosts else self._call("POST", "/hosts", json={"name": spec.name}) + host_id = int(host["id"]) + have = {_port_key(str(p.get("type", "")), str(p.get("address", ""))) + for p in host.get("ports") or []} + try: + for ptype, address in wanted: + if _port_key(ptype, address) in have: + continue + self._call("POST", f"/hosts/{host_id}/ports", + json={"type": ptype, "address": address}) + except ArrayError: + if created: + try: + self._call("DELETE", f"/hosts/{host_id}", params=_APPROVED, + missing_ok=True) + except ArrayError: + pass + raise + return str(host_id) + + def delete_host(self, host_id: str) -> None: + if self._int_id(host_id) is None: + return + self._call("DELETE", f"/hosts/{host_id}", params=_APPROVED, missing_ok=True) + + # -- mappings --------------------------------------------------------------- + + def list_mappings(self, volume_id: str) -> list[Mapping]: + luns = self._call("GET", f"/volumes/{volume_id}/luns") or [] + return [Mapping(volume_id=str(volume_id), host_id=str(m["host_id"]), lun=int(m["lun"])) + for m in luns if m.get("host_id") and not m.get("clustered")] + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + for m in self.list_mappings(volume_id): + if m.host_id == host_id: + return m + result = self._call("POST", f"/hosts/{host_id}/luns", + json={"volume_id": self._int_id(volume_id)}) + return Mapping(volume_id=str(volume_id), host_id=str(host_id), lun=int(result["lun"])) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + luns = self._call("GET", f"/volumes/{volume_id}/luns", missing_ok=True) or [] + if not any(str(m.get("host_id")) == str(host_id) for m in luns): + return + self._call("DELETE", f"/hosts/{host_id}/luns/volume_id/{volume_id}", + params=_APPROVED, missing_ok=True) diff --git a/simplyblock_core/storage_arrays/adapters/netapp_ontap.py b/simplyblock_core/storage_arrays/adapters/netapp_ontap.py new file mode 100644 index 0000000000..6f9587f867 --- /dev/null +++ b/simplyblock_core/storage_arrays/adapters/netapp_ontap.py @@ -0,0 +1,799 @@ +"""NetApp ONTAP 9.x REST API (AFF/FAS/ASA; also Lenovo ThinkSystem DM). + +Auth: HTTP basic (``username``/``password``) on every request; there is no +session to establish. ``login`` checks that the configured SVM exists. + +Layout (``volume_per_lun``, the default): every simplyblock volume is its +own FlexVol ``sb_`` (FlexVol names cannot contain ``-``) holding one +LUN or NVMe namespace ``sb-``. The FlexVol is thin (``guarantee none``), +has no snapshot reserve and no snapshot policy, and is sized +``flexvol_headroom_percent`` above the LUN so a fully written thin LUN does +not run its volume out of space. With ``volume_per_lun: false`` LUNs are +placed in one existing FlexVol (``flexvol``) that the operator sizes. + +Protocol family. ``create_volume`` has no protocol argument, so the option +``protocol_family`` (``scsi`` default, or ``nvme``) decides whether new +volumes are LUNs (``/api/storage/luns``) or namespaces +(``/api/storage/namespaces``). Existing volumes of either kind are always +found, mapped and deleted correctly. ``capabilities.protocols`` is a class +attribute, so it lists every protocol ONTAP serves; mapping a LUN to an NVMe +host (or a namespace to a SCSI host) raises :class:`ArrayUnsupportedError`. + +SCSI hosts are igroups (``protocol mixed``, ``os_type linux``); the host id +is the igroup name; volumes are mapped with ``lun-maps`` and the mapping's +``lun`` is the ``logical_unit_number``. Volume identity: ``naa.600a0980`` + +hex of the 12-character ASCII LUN serial number. + +NVMe hosts. ONTAP maps a namespace to exactly one subsystem, so a subsystem +cannot stand for a host if a volume is to be attached to several hosts. +Instead: + +* ``ensure_host`` creates a *host subsystem* named after the host holding + the host NQNs (and nothing else). The host id is its name. +* ``map_volume`` maps the namespace to its own *volume subsystem* + (``sb_``, comment :data:`VOLUME_SUBSYSTEM_COMMENT`, created on first + map) and adds the host's NQNs to it; ``unmap_volume`` removes them. +* The mapping's ``lun`` is the namespace's NSID in the volume subsystem. + +Volume subsystems are recognised by their comment and never count as hosts. +Namespaces carry no serial-derived identity in the REST API, so their +:attr:`Volume.identifier` is ``id.`` (the same UUID the host +sees as ``/dev/disk/by-id/nvme-uuid.``); ``nguid`` is left empty. + +ONTAP lets an initiator / host NQN sit in several igroups / subsystems. The +contract does not: before adding one, the adapter looks it up and raises +:class:`ArrayConflictError` if another host (igroup or host subsystem) has it. + +Asynchronous calls (FlexVol create/resize/delete) answer ``202`` with a job +link; the adapter polls ``/api/cluster/jobs/{uuid}`` until ``success`` or +``failure``, at most ``job_timeout_s`` seconds. + +Verify against a real system +---------------------------- +* FlexVol ``DELETE`` of an online volume without a junction path succeeds + without first taking it offline (the adapter does not offline it). +* A thin LUN/namespace may not be larger than its FlexVol (the fake + enforces this; it is why resize grows the FlexVol first). +* ``aggregates`` omitted on FlexVol ``POST`` lets ONTAP place the volume + (9.8+); with several ``aggregates`` configured the adapter picks the one + with the most ``space.block_storage.available`` (needs read access to + ``/api/storage/aggregates``, which SVM-scoped accounts may lack). +* Name/duplicate errors: HTTP 409 vs 400, and the numeric ONTAP error codes + (the adapter classifies by status and message text, not by code). +* FlexVol duplicate name surfaces as a failed job rather than a synchronous + error. +* ``PATCH`` of an igroup's ``protocol`` to ``mixed`` (only needed for igroups + not created by this adapter). +* ``nsid`` in subsystem-maps is a hex string such as ``"00000001h"``; the + subsystem-map ``DELETE`` path is + ``/subsystem-maps/{subsystem.uuid}/{namespace.uuid}``. +* ``allow_delete_with_hosts`` on subsystem ``DELETE``. +* Namespace resize via ``PATCH space.size`` (grow only). +* Whether query values match case-insensitively (WWPNs are always written in + lowercase colon form, IQNs in lowercase). +* ASA r2 systems (ONTAP 9.16+ "ASA r2") hide FlexVols; this adapter targets + the classic unified/ASA personality. +* NVMe/FC hosts are identified by NQN, not WWPN (see :meth:`_nqn`). +""" +from __future__ import annotations + +from collections.abc import Callable +from typing import Any +from urllib.parse import quote + +import requests + +from ..base import ( + ArrayAdapter, + ArrayConflictError, + ArrayError, + ArrayInfo, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + Capabilities, + HostSpec, + Initiator, + Mapping, + OptionDoc, + Protocol, + Volume, + colon_wwpn, +) +from ..http import json_of + +NETAPP_NAA_PREFIX = "naa.600a0980" +VOLUME_SUBSYSTEM_COMMENT = "simplyblock volume subsystem" +FLEXVOL_COMMENT = "simplyblock" +MIN_FLEXVOL_BYTES = 20 << 20 +_FAMILIES = ("scsi", "nvme") +_GUARANTEES = ("none", "volume") +_LUN = "lun" +_NS = "namespace" + +_LUN_FIELDS = "uuid,name,serial_number,space.size,location.logical_unit,location.volume,svm" +_NS_FIELDS = "uuid,name,space.size,location.namespace,location.volume,svm" + + +def ontap_wwn(serial: str) -> str: + """``naa.600a0980`` + hex of the 12-character ASCII LUN serial.""" + return NETAPP_NAA_PREFIX + serial.encode("ascii").hex() + + +def flexvol_name(name: str) -> str: + """FlexVol names allow letters, digits and ``_`` only.""" + return name.replace("-", "_") + + +def parse_nsid(value: Any) -> int | None: + """``"00000001h"`` (ONTAP's NSID format) or an int -> ``1``.""" + if value is None: + return None + if isinstance(value, int): + return value + text = str(value).strip().lower().removesuffix("h") + try: + return int(text, 16) + except ValueError: + return None + + +def _as_bool(value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() in ("1", "true", "yes", "on") + return bool(value) + + +class NetAppOntapAdapter(ArrayAdapter): + vendor = "netapp-ontap" + display_name = "NetApp ONTAP" + capabilities = Capabilities( + protocols=frozenset({Protocol.ISCSI, Protocol.FC, Protocol.NVME_TCP, Protocol.NVME_FC}), + size_granularity_bytes=4096, + min_size_bytes=1 << 20, + max_name_len=96, + ) + credential_fields = (frozenset({"username", "password"}),) + option_docs = ( + OptionDoc("svm", "Storage VM (vserver) that owns the volumes, igroups and " + "subsystems.", required=True), + OptionDoc("protocol_family", "'scsi' (LUNs, for iSCSI/FC hosts) or 'nvme' " + "(namespaces, for NVMe/TCP and NVMe/FC hosts): what new volumes " + "are created as.", default="scsi"), + OptionDoc("aggregates", "Aggregate name(s) for new FlexVols (list or comma " + "separated); with several, the one with most free space is used. " + "Default: ONTAP chooses."), + OptionDoc("volume_per_lun", "Create one FlexVol per LUN/namespace.", default=True), + OptionDoc("flexvol", "Existing FlexVol for all LUNs/namespaces; required when " + "volume_per_lun is false."), + OptionDoc("space_guarantee", "FlexVol space guarantee: 'none' (thin) or " + "'volume'.", default="none"), + OptionDoc("snapshot_reserve_percent", "FlexVol snapshot reserve (0-90).", default=0), + OptionDoc("snapshot_policy", "FlexVol snapshot policy.", default="none"), + OptionDoc("flexvol_headroom_percent", "FlexVol size above the LUN/namespace size, " + "for metadata.", default=5), + OptionDoc("os_type", "os_type of LUNs, namespaces, igroups and subsystems.", + default="linux"), + OptionDoc("namespace_block_size", "NVMe namespace block size (512 or 4096).", + default=4096), + OptionDoc("job_timeout_s", "How long to wait for an asynchronous ONTAP job.", + default=300), + ) + + def __init__(self, ctx): + super().__init__(ctx) + self._sleep: Callable[[float], None] = self.session.sleep + self.svm = str(self.option("svm")).strip() + if not self.svm: + raise ArrayError("NetApp ONTAP option svm must not be empty") + self.family = str(self.option("protocol_family")).strip().lower() + if self.family not in _FAMILIES: + raise ArrayError(f"NetApp ONTAP protocol_family must be one of {_FAMILIES}, " + f"got {self.family!r}") + self.volume_per_lun = _as_bool(self.option("volume_per_lun")) + self.shared_flexvol = self.option("flexvol") + if not self.volume_per_lun and not self.shared_flexvol: + raise ArrayError("NetApp ONTAP option flexvol is required when volume_per_lun " + "is false") + aggr = self.option("aggregates") or [] + if isinstance(aggr, str): + aggr = [a for a in (x.strip() for x in aggr.split(",")) if a] + self.aggregates: list[str] = [str(a) for a in aggr] + self.guarantee = str(self.option("space_guarantee")) + if self.guarantee not in _GUARANTEES: + raise ArrayError(f"NetApp ONTAP space_guarantee must be one of {_GUARANTEES}") + self.snapshot_reserve = self._int_option("snapshot_reserve_percent", 0, 90) + self.headroom = self._int_option("flexvol_headroom_percent", 0, 100) + self.block_size = self._int_option("namespace_block_size", 512, 4096) + if self.block_size not in (512, 4096): + raise ArrayError("NetApp ONTAP namespace_block_size must be 512 or 4096") + self.job_timeout_s = float(self._int_option("job_timeout_s", 1, 86400)) + self.os_type = str(self.option("os_type")) + + def _int_option(self, name: str, lo: int, hi: int) -> int: + raw = self.option(name) + try: + value = int(raw) + except (TypeError, ValueError): + raise ArrayError(f"NetApp ONTAP option {name} must be an integer") from None + if not lo <= value <= hi: + raise ArrayError(f"NetApp ONTAP option {name} must be in {lo}..{hi}") + return value + + # -- session ------------------------------------------------------------ + + def login(self) -> None: + self.session.set_basic_auth(self.secret("username"), self.secret("password")) + if not self._records("/api/svm/svms", {"name": self.svm, "fields": "uuid,name"}): + raise ArrayNotFoundError(f"SVM {self.svm!r} not found on array {self.config.id}") + + def close(self) -> None: + self.session.clear_auth() + + # -- request helpers ----------------------------------------------------- + + def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None) -> requests.Response: + """One request; ONTAP 400 errors are classified by message.""" + resp = self.session.request(method, path, params=params, json=json, ok=(400,)) + if resp.status_code == 400: + raise self._error(method, path, json_of(resp) or {}) + return resp + + @staticmethod + def _error(method: str, path: str, body: dict[str, Any]) -> ArrayError: + err = body.get("error") or {} + message = str(err.get("message", "")) or "request failed" + code = err.get("code", "") + text = f"{method} {path}: {message} (ONTAP error {code})" + lowered = message.lower() + if "doesn't exist" in lowered or "does not exist" in lowered or "not found" in lowered: + return ArrayNotFoundError(text) + if any(w in lowered for w in ("already", "duplicate", "in use", "is mapped")): + return ArrayConflictError(text) + return ArrayRequestError(text, status=400) + + def _records(self, path: str, params: dict[str, Any]) -> list[dict[str, Any]]: + """GET a collection, following ``_links.next``.""" + out: list[dict[str, Any]] = [] + next_path: str | None = path + query: dict[str, Any] | None = params + while next_path: + body = json_of(self._call("GET", next_path, params=query)) or {} + out.extend(body.get("records", [])) + next_path = ((body.get("_links") or {}).get("next") or {}).get("href") + query = None + return out + + def _one(self, path: str, params: dict[str, Any]) -> dict[str, Any] | None: + records = self._records(path, params) + return records[0] if records else None + + def _async(self, method: str, path: str, *, params: dict[str, Any] | None = None, + json: Any = None) -> dict[str, Any]: + """Issue a request that may answer 202 + job, and wait for the job.""" + resp = self._call(method, path, params=params, json=json) + body = json_of(resp) or {} + job = body.get("job") + if job: + self._wait_job(job, f"{method} {path}") + return body + + def _wait_job(self, job: dict[str, Any], what: str) -> None: + href = ((job.get("_links") or {}).get("self") or {}).get("href") \ + or f"/api/cluster/jobs/{job['uuid']}" + waited = 0.0 + interval = 0.5 + while True: + state = json_of(self._call("GET", href, + params={"fields": "uuid,state,message,code"})) or {} + status = state.get("state") + if status == "success": + return + if status == "failure": + raise self._error_from_job(what, state) + if waited >= self.job_timeout_s: + raise ArrayRequestError( + f"{what}: ONTAP job {job.get('uuid')} still {status} after " + f"{self.job_timeout_s:.0f}s") + self._sleep(interval) + waited += interval + interval = min(interval * 2, 5.0) + + def _error_from_job(self, what: str, state: dict[str, Any]) -> ArrayError: + method, _, path = what.partition(" ") + message = f"{state.get('message') or 'job failed'} [job {state.get('uuid', '')}]" + return self._error(method, path, {"error": {"message": message, + "code": state.get("code", "")}}) + + @staticmethod + def _created(resp_body: dict[str, Any]) -> dict[str, Any] | None: + records = resp_body.get("records") or [] + return records[0] if records else None + + def _svm_ref(self) -> dict[str, str]: + return {"name": self.svm} + + # -- volumes: lookup ----------------------------------------------------- + + def _volume_from_lun(self, rec: dict[str, Any]) -> Volume: + serial = rec.get("serial_number") or "" + loc = rec.get("location") or {} + name = loc.get("logical_unit") or str(rec.get("name", "")).rsplit("/", 1)[-1] + return Volume(id=rec["uuid"], name=name, size_bytes=int(rec["space"]["size"]), + wwn=ontap_wwn(serial) if serial else None) + + @staticmethod + def _volume_from_ns(rec: dict[str, Any]) -> Volume: + loc = rec.get("location") or {} + name = loc.get("namespace") or str(rec.get("name", "")).rsplit("/", 1)[-1] + return Volume(id=rec["uuid"], name=name, size_bytes=int(rec["space"]["size"])) + + def _kinds(self) -> tuple[str, str]: + return (_LUN, _NS) if self.family == "scsi" else (_NS, _LUN) + + def _lookup(self, kind: str, params: dict[str, Any]) -> dict[str, Any] | None: + if kind == _LUN: + return self._one("/api/storage/luns", {**params, "svm.name": self.svm, + "fields": _LUN_FIELDS}) + return self._one("/api/storage/namespaces", {**params, "svm.name": self.svm, + "fields": _NS_FIELDS}) + + def _locate(self, volume_id: str) -> tuple[str, dict[str, Any]] | None: + for kind in self._kinds(): + rec = self._lookup(kind, {"uuid": volume_id}) + if rec is not None: + return kind, rec + return None + + def _require(self, volume_id: str) -> tuple[str, dict[str, Any]]: + found = self._locate(volume_id) + if found is None: + raise ArrayNotFoundError( + f"Volume {volume_id} not found in SVM {self.svm} on array {self.config.id}") + return found + + def _to_volume(self, kind: str, rec: dict[str, Any]) -> Volume: + return self._volume_from_lun(rec) if kind == _LUN else self._volume_from_ns(rec) + + def _flexvol_for(self, name: str) -> str: + return flexvol_name(name) if self.volume_per_lun else str(self.shared_flexvol) + + # -- contract ------------------------------------------------------------- + + def get_info(self) -> ArrayInfo: + cluster = json_of(self._call("GET", "/api/cluster", + params={"fields": "name,uuid,version"})) or {} + model = "" + try: + nodes = self._records("/api/cluster/nodes", {"fields": "name,model,serial_number"}) + model = next((n["model"] for n in nodes if n.get("model")), "") + except (ArrayRequestError, ArrayNotFoundError): + pass + version = (cluster.get("version") or {}).get("full", "") + return ArrayInfo(vendor=self.vendor, model=model, version=version, + serial=cluster.get("uuid", "")) + + def get_volume(self, volume_id: str) -> Volume | None: + found = self._locate(volume_id) + return self._to_volume(*found) if found else None + + def find_volume(self, name: str) -> Volume | None: + path = f"/vol/{self._flexvol_for(name)}/{name}" + for kind in self._kinds(): + rec = self._lookup(kind, {"name": path}) + if rec is not None: + return self._to_volume(kind, rec) + return None + + def _flexvol_bytes(self, size_bytes: int) -> int: + usable = -(-size_bytes * (100 + self.headroom) // 100) + total = -(-usable * 100 // (100 - self.snapshot_reserve)) + total = -(-total // 4096) * 4096 + return max(total, MIN_FLEXVOL_BYTES) + + def _get_flexvol(self, name: str) -> dict[str, Any] | None: + return self._one("/api/storage/volumes", + {"name": name, "svm.name": self.svm, "fields": "uuid,name,size"}) + + def _pick_aggregate(self) -> list[dict[str, str]]: + if not self.aggregates: + return [] + if len(self.aggregates) == 1: + return [{"name": self.aggregates[0]}] + # More than one aggregate in the POST would make a FlexGroup: choose one. + recs = self._records("/api/storage/aggregates", + {"name": "|".join(self.aggregates), + "fields": "name,space.block_storage.available"}) + if not recs: + raise ArrayNotFoundError(f"None of the aggregates {self.aggregates} exist") + best = max(recs, key=lambda r: int( + ((r.get("space") or {}).get("block_storage") or {}).get("available", 0))) + return [{"name": best["name"]}] + + def _create_flexvol(self, name: str, size_bytes: int) -> None: + body: dict[str, Any] = { + "name": name, + "svm": self._svm_ref(), + "size": size_bytes, + "type": "rw", + "guarantee": {"type": self.guarantee}, + "space": {"snapshot": {"reserve_percent": self.snapshot_reserve}}, + "snapshot_policy": {"name": str(self.option("snapshot_policy"))}, + "comment": FLEXVOL_COMMENT, + } + aggrs = self._pick_aggregate() + if aggrs: + body["aggregates"] = aggrs + self._async("POST", "/api/storage/volumes", json=body) + + def _grow_flexvol(self, vol: dict[str, Any], size_bytes: int) -> None: + if int(vol.get("size", 0)) < size_bytes: + self._async("PATCH", f"/api/storage/volumes/{vol['uuid']}", + json={"size": size_bytes}) + + def _delete_flexvol(self, uuid: str) -> None: + try: + self._async("DELETE", f"/api/storage/volumes/{uuid}") + except ArrayNotFoundError: + pass + + def create_volume(self, name: str, size_bytes: int) -> Volume: + flex = self._flexvol_for(name) + created_flexvol: str | None = None + if self.volume_per_lun: + want = self._flexvol_bytes(size_bytes) + vol = self._get_flexvol(flex) + if vol is None: # a re-run may find the FlexVol of an interrupted create + self._create_flexvol(flex, want) + vol = self._get_flexvol(flex) + if vol is None: + raise ArrayRequestError(f"FlexVol {flex} missing after create") + created_flexvol = vol["uuid"] + else: + self._grow_flexvol(vol, want) + path = f"/vol/{flex}/{name}" + kind = _LUN if self.family == "scsi" else _NS + try: + if kind == _LUN: + self._call("POST", "/api/storage/luns", params={"return_records": "true"}, json={ + "svm": self._svm_ref(), "name": path, "os_type": self.os_type, + "space": {"size": size_bytes, "guarantee": {"requested": False}}, + }) + else: + self._call("POST", "/api/storage/namespaces", params={"return_records": "true"}, + json={"svm": self._svm_ref(), "name": path, "os_type": self.os_type, + "space": {"size": size_bytes, + "block_size": self.block_size}}) + except ArrayError: + if created_flexvol: + try: + self._delete_flexvol(created_flexvol) + except ArrayError: + pass + raise + rec = self._lookup(kind, {"name": path}) + if rec is None: + raise ArrayRequestError(f"{kind} {path} missing after create") + return self._to_volume(kind, rec) + + def _owns_flexvol(self, kind: str, rec: dict[str, Any]) -> bool: + """True when the LUN/namespace sits in its own per-volume FlexVol.""" + flex = ((rec.get("location") or {}).get("volume") or {}).get("name", "") + name = self._to_volume(kind, rec).name + return bool(flex) and flex != self.shared_flexvol and flex == flexvol_name(name) + + def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: + kind, rec = self._require(volume_id) + loc_vol = (rec.get("location") or {}).get("volume") or {} + if self._owns_flexvol(kind, rec): + vol = self._get_flexvol(loc_vol.get("name", "")) + if vol is None: + raise ArrayNotFoundError(f"FlexVol of volume {volume_id} not found") + self._grow_flexvol(vol, self._flexvol_bytes(size_bytes)) + base = "/api/storage/luns" if kind == _LUN else "/api/storage/namespaces" + self._call("PATCH", f"{base}/{volume_id}", json={"space": {"size": size_bytes}}) + kind, rec = self._require(volume_id) + return self._to_volume(kind, rec) + + def delete_volume(self, volume_id: str) -> None: + kind, rec = self._require(volume_id) + loc_vol = (rec.get("location") or {}).get("volume") or {} + if kind == _LUN: + self._call("DELETE", f"/api/storage/luns/{volume_id}") + else: + smap = self._subsystem_map(volume_id) + if smap is not None: + sub = smap["subsystem"] + self._call("DELETE", + f"/api/protocols/nvme/subsystem-maps/{sub['uuid']}/{volume_id}") + self._delete_subsystem(sub["uuid"]) + self._call("DELETE", f"/api/storage/namespaces/{volume_id}") + # Only a FlexVol of our own (one per LUN) is removed; never the shared one. + flex = loc_vol.get("name", "") + if self._owns_flexvol(kind, rec): + uuid = loc_vol.get("uuid") or (self._get_flexvol(flex) or {}).get("uuid") + if uuid: + self._delete_flexvol(uuid) + + # -- hosts ---------------------------------------------------------------- + + @staticmethod + def _nqn(i: Initiator) -> str: + if i.protocol == Protocol.NVME_FC and ":" in i.address and not i.address.startswith( + "nqn."): + raise ArrayUnsupportedError( + "ONTAP identifies NVMe/FC hosts by host NQN, not by WWPN: " + f"{i.address!r}") + return i.address + + def ensure_host(self, spec: HostSpec) -> str: + if all(i.protocol.is_nvme for i in spec.initiators): + return self._ensure_nvme_host(spec) + if all(i.protocol.is_scsi for i in spec.initiators): + return self._ensure_igroup(spec) + raise ArrayUnsupportedError( + f"NetApp ONTAP adapter does not support {sorted(p.value for p in spec.protocols)}") + + # SCSI: igroups + + @staticmethod + def _igroup_initiator(i: Initiator) -> str: + return colon_wwpn(i.address) if i.protocol == Protocol.FC else i.normalized() + + @staticmethod + def _norm_initiator(value: str) -> str: + v = value.strip().lower() + if v.startswith(("iqn.", "eui.", "naa.")): + return v + return v.replace(":", "").replace("-", "").removeprefix("0x") + + def _igroup(self, name: str) -> dict[str, Any] | None: + return self._one("/api/protocols/san/igroups", + {"name": name, "svm.name": self.svm, + "fields": "uuid,name,protocol,os_type,initiators.name"}) + + def _ensure_igroup(self, spec: HostSpec) -> str: + wanted = [self._igroup_initiator(i) for i in spec.initiators] + owners = self._records("/api/protocols/san/igroups", + {"svm.name": self.svm, "initiators.name": "|".join(wanted), + "fields": "uuid,name,initiators.name"}) + want_norm = {self._norm_initiator(w) for w in wanted} + for g in owners: + if g["name"] == spec.name: + continue + taken = want_norm & {self._norm_initiator(i["name"]) + for i in g.get("initiators") or []} + if taken: + raise ArrayConflictError( + f"Initiator(s) {', '.join(sorted(taken))} already belong to igroup " + f"{g['name']!r} in SVM {self.svm}") + group = self._igroup(spec.name) + if group is None: + self._call("POST", "/api/protocols/san/igroups", + params={"return_records": "true"}, + json={"svm": self._svm_ref(), "name": spec.name, "os_type": self.os_type, + "protocol": "mixed", + "initiators": [{"name": w} for w in wanted]}) + return spec.name + have = {self._norm_initiator(i["name"]) for i in group.get("initiators") or []} + missing = [w for w in wanted if self._norm_initiator(w) not in have] + if missing: + needs = {"iscsi" if i.protocol == Protocol.ISCSI else "fcp" + for i in spec.initiators + if self._norm_initiator(self._igroup_initiator(i)) not in have} + if group.get("protocol") not in ("mixed", *needs) or len(needs) > 1: + self._call("PATCH", f"/api/protocols/san/igroups/{group['uuid']}", + json={"protocol": "mixed"}) + self._call("POST", f"/api/protocols/san/igroups/{group['uuid']}/initiators", + json={"records": [{"name": m} for m in missing]}) + return spec.name + + # NVMe: host subsystems + + def _subsystems(self, params: dict[str, Any]) -> list[dict[str, Any]]: + return self._records("/api/protocols/nvme/subsystems", + {**params, "svm.name": self.svm, + "fields": "uuid,name,comment,os_type,hosts.nqn"}) + + @staticmethod + def _is_volume_subsystem(sub: dict[str, Any]) -> bool: + return sub.get("comment") == VOLUME_SUBSYSTEM_COMMENT + + def _host_subsystem(self, name: str) -> dict[str, Any] | None: + subs = [s for s in self._subsystems({"name": name}) if not self._is_volume_subsystem(s)] + return subs[0] if subs else None + + def _ensure_nvme_host(self, spec: HostSpec) -> str: + wanted = [self._nqn(i) for i in spec.initiators] + for other in self._subsystems({"hosts.nqn": "|".join(wanted)}): + if other["name"] == spec.name or self._is_volume_subsystem(other): + continue + taken = set(wanted) & {h["nqn"] for h in other.get("hosts") or []} + if taken: + raise ArrayConflictError( + f"Host NQN(s) {', '.join(sorted(taken))} already belong to subsystem " + f"{other['name']!r} in SVM {self.svm}") + sub = self._host_subsystem(spec.name) + if sub is None: + self._call("POST", "/api/protocols/nvme/subsystems", + params={"return_records": "true"}, + json={"svm": self._svm_ref(), "name": spec.name, "os_type": self.os_type, + "hosts": [{"nqn": n} for n in wanted]}) + return spec.name + have = {h["nqn"] for h in sub.get("hosts") or []} + missing = [n for n in wanted if n not in have] + if missing: + self._add_subsystem_hosts(sub["uuid"], missing) + return spec.name + + def _add_subsystem_hosts(self, sub_uuid: str, nqns: list[str]) -> None: + self._call("POST", f"/api/protocols/nvme/subsystems/{sub_uuid}/hosts", + json={"records": [{"nqn": n} for n in nqns]}) + + def _delete_subsystem(self, sub_uuid: str) -> None: + try: + self._call("DELETE", f"/api/protocols/nvme/subsystems/{sub_uuid}", + params={"allow_delete_with_hosts": "true"}) + except ArrayNotFoundError: + pass + + def delete_host(self, host_id: str) -> None: + group = self._igroup(host_id) + if group is not None: + try: + self._call("DELETE", f"/api/protocols/san/igroups/{group['uuid']}") + except ArrayNotFoundError: + pass + sub = self._host_subsystem(host_id) + if sub is not None: + self._delete_subsystem(sub["uuid"]) + + # -- mappings ------------------------------------------------------------- + + def _lun_maps(self, lun_uuid: str, igroup: str | None = None) -> list[dict[str, Any]]: + params: dict[str, Any] = {"lun.uuid": lun_uuid, "svm.name": self.svm, + "fields": "lun.uuid,igroup.uuid,igroup.name," + "logical_unit_number"} + if igroup is not None: + params["igroup.name"] = igroup + return self._records("/api/protocols/san/lun-maps", params) + + @staticmethod + def _lun_mapping(rec: dict[str, Any]) -> Mapping: + return Mapping(volume_id=rec["lun"]["uuid"], host_id=rec["igroup"]["name"], + lun=rec.get("logical_unit_number")) + + def _subsystem_map(self, ns_uuid: str) -> dict[str, Any] | None: + return self._one("/api/protocols/nvme/subsystem-maps", + {"namespace.uuid": ns_uuid, "svm.name": self.svm, + "fields": "namespace.uuid,subsystem.uuid,subsystem.name,nsid"}) + + def _volume_subsystem_hosts(self, smap: dict[str, Any]) -> dict[str, Any]: + sub = smap["subsystem"] + recs = self._subsystems({"uuid": sub["uuid"]}) + if not recs: + raise ArrayNotFoundError(f"Subsystem {sub.get('name')} not found") + return recs[0] + + def _nvme_mappings(self, ns_uuid: str) -> list[Mapping]: + smap = self._subsystem_map(ns_uuid) + if smap is None: + return [] + vsub = self._volume_subsystem_hosts(smap) + nqns = {h["nqn"] for h in vsub.get("hosts") or []} + if not nqns: + return [] + nsid = parse_nsid(smap.get("nsid")) + out: list[Mapping] = [] + for sub in self._subsystems({"hosts.nqn": "|".join(sorted(nqns))}): + if self._is_volume_subsystem(sub): + continue + if nqns & {h["nqn"] for h in sub.get("hosts") or []}: + out.append(Mapping(volume_id=ns_uuid, host_id=sub["name"], lun=nsid)) + return out + + def map_volume(self, volume_id: str, host_id: str) -> Mapping: + kind, rec = self._require(volume_id) + if kind == _LUN: + return self._map_lun(volume_id, host_id) + return self._map_namespace(volume_id, rec, host_id) + + def _map_lun(self, lun_uuid: str, host_id: str) -> Mapping: + if self._igroup(host_id) is None: + if self._host_subsystem(host_id) is not None: + raise ArrayUnsupportedError( + f"Host {host_id} is an NVMe host; volume {lun_uuid} is a SCSI LUN " + "(protocol_family scsi)") + raise ArrayNotFoundError(f"igroup {host_id} not found in SVM {self.svm}") + existing = self._lun_maps(lun_uuid, host_id) + if existing: + return self._lun_mapping(existing[0]) + self._call("POST", "/api/protocols/san/lun-maps", params={"return_records": "true"}, + json={"svm": self._svm_ref(), "lun": {"uuid": lun_uuid}, + "igroup": {"name": host_id}}) + created = self._lun_maps(lun_uuid, host_id) + if not created: + raise ArrayRequestError(f"lun-map {lun_uuid} -> {host_id} missing after create") + return self._lun_mapping(created[0]) + + def _map_namespace(self, ns_uuid: str, rec: dict[str, Any], host_id: str) -> Mapping: + host = self._host_subsystem(host_id) + if host is None: + if self._igroup(host_id) is not None: + raise ArrayUnsupportedError( + f"Host {host_id} is a SCSI host; volume {ns_uuid} is an NVMe namespace " + "(protocol_family nvme)") + raise ArrayNotFoundError(f"NVMe host subsystem {host_id} not found in SVM {self.svm}") + host_nqns = [h["nqn"] for h in host.get("hosts") or []] + smap = self._subsystem_map(ns_uuid) + if smap is None: + sub_name = flexvol_name(self._volume_from_ns(rec).name) + vsubs = [s for s in self._subsystems({"name": sub_name}) + if self._is_volume_subsystem(s)] + if vsubs: + sub_uuid = vsubs[0]["uuid"] + else: + body = json_of(self._call( + "POST", "/api/protocols/nvme/subsystems", params={"return_records": "true"}, + json={"svm": self._svm_ref(), "name": sub_name, "os_type": self.os_type, + "comment": VOLUME_SUBSYSTEM_COMMENT})) or {} + created = self._created(body) + if created is None: + created = next(iter(s for s in self._subsystems({"name": sub_name}) + if self._is_volume_subsystem(s)), None) + if created is None: + raise ArrayRequestError(f"Subsystem {sub_name} missing after create") + sub_uuid = created["uuid"] + self._call("POST", "/api/protocols/nvme/subsystem-maps", + params={"return_records": "true"}, + json={"svm": self._svm_ref(), "namespace": {"uuid": ns_uuid}, + "subsystem": {"uuid": sub_uuid}}) + smap = self._subsystem_map(ns_uuid) + if smap is None: + raise ArrayRequestError(f"subsystem-map of {ns_uuid} missing after create") + vsub = self._volume_subsystem_hosts(smap) + if not self._is_volume_subsystem(vsub): + raise ArrayConflictError( + f"Namespace {ns_uuid} is mapped to subsystem {vsub['name']!r}, which " + "simplyblock does not manage") + have = {h["nqn"] for h in vsub.get("hosts") or []} + missing = [n for n in host_nqns if n not in have] + if missing: + self._add_subsystem_hosts(vsub["uuid"], missing) + return Mapping(volume_id=ns_uuid, host_id=host_id, lun=parse_nsid(smap.get("nsid"))) + + def unmap_volume(self, volume_id: str, host_id: str) -> None: + found = self._locate(volume_id) + if found is None: + return + kind, _ = found + if kind == _LUN: + for m in self._lun_maps(volume_id, host_id): + try: + self._call("DELETE", f"/api/protocols/san/lun-maps/{volume_id}/" + f"{m['igroup']['uuid']}") + except ArrayNotFoundError: + pass + return + smap = self._subsystem_map(volume_id) + host = self._host_subsystem(host_id) + if smap is None or host is None: + return + vsub = self._volume_subsystem_hosts(smap) + if not self._is_volume_subsystem(vsub): + return + have = {h["nqn"] for h in vsub.get("hosts") or []} + for h in host.get("hosts") or []: + if h["nqn"] in have: + try: + self._call("DELETE", f"/api/protocols/nvme/subsystems/{vsub['uuid']}/hosts/" + f"{quote(h['nqn'], safe='')}") + except ArrayNotFoundError: + pass + + def list_mappings(self, volume_id: str) -> list[Mapping]: + kind, _ = self._require(volume_id) + if kind == _LUN: + return [self._lun_mapping(r) for r in self._lun_maps(volume_id)] + return self._nvme_mappings(volume_id) diff --git a/simplyblock_core/storage_arrays/adapters/pure.py b/simplyblock_core/storage_arrays/adapters/pure.py index 5c2ed4864e..866d3894da 100644 --- a/simplyblock_core/storage_arrays/adapters/pure.py +++ b/simplyblock_core/storage_arrays/adapters/pure.py @@ -195,6 +195,10 @@ def ensure_host(self, spec: HostSpec) -> str: return spec.name def delete_host(self, host_id: str) -> None: + if self._host(host_id) is None: + return + if self._items("GET", "/connections", params={"host_names": host_id}, missing_ok=True): + raise ArrayConflictError(f"Host {host_id} still has volume connections") self._items("DELETE", "/hosts", params={"names": host_id}, missing_ok=True) def _connections(self, volume_id: str, host_id: str | None = None) -> list[Mapping]: diff --git a/simplyblock_core/storage_arrays/base.py b/simplyblock_core/storage_arrays/base.py index 0619d402e6..8d2a6b927e 100644 --- a/simplyblock_core/storage_arrays/base.py +++ b/simplyblock_core/storage_arrays/base.py @@ -105,7 +105,9 @@ def _strip(cls, v: str) -> str: return v.strip() def normalized(self) -> str: - if self.protocol in (Protocol.FC, Protocol.NVME_FC): + # NVMe/FC hosts are identified by their host NQN like every NVMe + # transport; only SCSI over FC uses WWPNs. + if self.protocol == Protocol.FC: return normalize_wwpn(self.address) if self.protocol == Protocol.ISCSI: return self.address.lower() @@ -146,8 +148,10 @@ class Volume(BaseModel): id: str name: str size_bytes: int - #: SCSI identity as the host sees it, ``naa.`` (lowercase). This is - #: what matches the array volume to ``/dev/disk/by-id`` on the host. + #: SCSI identity (VPD page 0x83 designator) as the host sees it: + #: ``naa.`` for most arrays, ``eui.`` where the array presents an + #: EUI-64 designator (HPE Nimble). Lowercase. This matches the array volume + #: to ``/dev/disk/by-id`` / the multipath WWID on the host. wwn: str | None = None #: NVMe identity, ``eui.<32 hex>`` (NGUID) where the array exposes one. nguid: str | None = None @@ -217,6 +221,9 @@ class Capabilities: max_name_len: int = 63 #: Whether resize can only grow (true for every supported array today). grow_only: bool = True + #: Most initiators one host may have; ``None`` = unlimited. PowerFlex SDC + #: hosts are exactly one client, so 1 there. + max_initiators_per_host: int | None = None @dataclass diff --git a/simplyblock_core/storage_arrays/client.py b/simplyblock_core/storage_arrays/client.py index e5bb928eab..cccef18751 100644 --- a/simplyblock_core/storage_arrays/client.py +++ b/simplyblock_core/storage_arrays/client.py @@ -154,11 +154,20 @@ def _call(self, op: str, fn: Callable[[ArrayAdapter], T]) -> T: # -- checks ----------------------------------------------------------------- - def _owned(self, vol: Volume, expected_identifier: str | None, action: str) -> None: + def _owned(self, vol: Volume, expected_identifier: str | None, action: str, + volume_uuid: str | None = None) -> None: + """Refuse to touch a volume simplyblock did not create, or that is not + the one it recorded. The name check matters where the array reuses + identities: an LDEV number / volume id (and the WWN derived from it) + can come back after a delete, the name derived from our UUID cannot.""" if not vol.name.startswith(MANAGED_PREFIX): raise ArrayConflictError( f"Refusing to {action} volume {vol.name!r} on array {self.config.id}: " "not created by simplyblock") + if volume_uuid is not None and vol.name != self.volume_name(volume_uuid): + raise ArrayConflictError( + f"Refusing to {action} volume {vol.name!r} on array {self.config.id}: " + f"name does not match recorded volume {volume_uuid}") if expected_identifier is not None and vol.identifier.lower() != expected_identifier.lower(): raise ArrayConflictError( f"Refusing to {action} volume {vol.name!r} on array {self.config.id}: " @@ -200,7 +209,8 @@ def find_volume(self, volume_uuid: str) -> Volume | None: return self._call("find_volume", lambda a: a.find_volume(name)) def resize_volume(self, volume_id: str, size_bytes: int, - expected_identifier: str | None = None) -> Volume: + expected_identifier: str | None = None, + volume_uuid: str | None = None) -> Volume: caps = self.capabilities size = round_size(size_bytes, caps.size_granularity_bytes, caps.min_size_bytes) @@ -208,7 +218,7 @@ def op(a: ArrayAdapter) -> Volume: vol = a.get_volume(volume_id) if vol is None: raise ArrayError(f"Volume {volume_id} not found on array {self.config.id}") - self._owned(vol, expected_identifier, "resize") + self._owned(vol, expected_identifier, "resize", volume_uuid) if size == vol.size_bytes: return vol if size < vol.size_bytes and caps.grow_only: @@ -221,13 +231,17 @@ def op(a: ArrayAdapter) -> Volume: return self._call("resize_volume", op) - def delete_volume(self, volume_id: str, expected_identifier: str | None = None) -> bool: - """Unmap from every host and delete. Returns False if already gone.""" + def delete_volume(self, volume_id: str, expected_identifier: str | None = None, + volume_uuid: str | None = None) -> bool: + """Unmap from every host and delete. Returns False if already gone. + + Callers should pass both ``expected_identifier`` and ``volume_uuid`` + from their record; together they pin the exact volume.""" def op(a: ArrayAdapter) -> bool: vol = a.get_volume(volume_id) if vol is None: return False - self._owned(vol, expected_identifier, "delete") + self._owned(vol, expected_identifier, "delete", volume_uuid) for m in a.list_mappings(vol.id): a.unmap_volume(vol.id, m.host_id) a.delete_volume(vol.id) @@ -242,6 +256,11 @@ def ensure_host(self, spec: HostSpec) -> str: raise ArrayError( f"{self.adapter_cls.display_name} adapter does not support " + ", ".join(sorted(p.value for p in unsupported))) + limit = self.capabilities.max_initiators_per_host + if limit is not None and len(spec.initiators) > limit: + raise ArrayError( + f"{self.adapter_cls.display_name} hosts take at most {limit} initiator(s); " + f"got {len(spec.initiators)}") if len(spec.name) > self.capabilities.max_name_len: raise ArrayError(f"Host name {spec.name!r} exceeds " f"{self.capabilities.max_name_len} characters") @@ -251,12 +270,13 @@ def delete_host(self, host_id: str) -> None: self._call("delete_host", lambda a: a.delete_host(host_id)) def map_volume(self, volume_id: str, host_id: str, - expected_identifier: str | None = None) -> Mapping: + expected_identifier: str | None = None, + volume_uuid: str | None = None) -> Mapping: def op(a: ArrayAdapter) -> Mapping: vol = a.get_volume(volume_id) if vol is None: raise ArrayError(f"Volume {volume_id} not found on array {self.config.id}") - self._owned(vol, expected_identifier, "map") + self._owned(vol, expected_identifier, "map", volume_uuid) return a.map_volume(vol.id, host_id) return self._call("map_volume", op) diff --git a/simplyblock_core/storage_arrays/http.py b/simplyblock_core/storage_arrays/http.py index 71c29fdc30..0f4e0819d9 100644 --- a/simplyblock_core/storage_arrays/http.py +++ b/simplyblock_core/storage_arrays/http.py @@ -56,7 +56,9 @@ def __init__( self.timeout_s = timeout_s self.retries = retries self.backoff_s = backoff_s - self._sleep = sleep + #: Used for retry backoff and by adapters that poll async jobs, so + #: tests can make all waiting instant. + self.sleep = sleep self.http = requests.Session() self.http.verify = verify self.http.headers.update({"Accept": "application/json"}) @@ -125,12 +127,12 @@ def request( except requests.RequestException as e: last_exc = e if attempt + 1 < attempts: - self._sleep(self.backoff_s * (attempt + 1)) + self.sleep(self.backoff_s * (attempt + 1)) continue raise ArrayRequestError( f"{method} {path}: {type(e).__name__}: {_safe(str(e))}") from e if resp.status_code in _RETRY_STATUSES and attempt + 1 < attempts: - self._sleep(self.backoff_s * (attempt + 1)) + self.sleep(self.backoff_s * (attempt + 1)) continue if 200 <= resp.status_code < 300 or resp.status_code in ok_extra: return resp diff --git a/simplyblock_core/storage_arrays/testing/__init__.py b/simplyblock_core/storage_arrays/testing/__init__.py index b295365650..67f49ad7e2 100644 --- a/simplyblock_core/storage_arrays/testing/__init__.py +++ b/simplyblock_core/storage_arrays/testing/__init__.py @@ -20,10 +20,12 @@ from __future__ import annotations import base64 +import http.client import io import json import re import threading +import types from collections.abc import Callable from dataclasses import dataclass, field from typing import Any @@ -193,12 +195,25 @@ def _encode(reply: Reply) -> tuple[bytes, dict[str, str]]: return json.dumps(reply.body).encode(), headers +class _Raw(io.BytesIO): + """Body stream that also carries the headers the way urllib3 does, so + ``requests`` stores Set-Cookie values in the session's cookie jar exactly + as it would for a real response.""" + + def __init__(self, content: bytes, headers: dict[str, str]): + super().__init__(content) + msg = http.client.HTTPMessage() + for k, v in headers.items(): + msg[k] = v + self._original_response = types.SimpleNamespace(msg=msg) + + def _to_response(request, reply: Reply) -> requests.Response: content, headers = _encode(reply) resp = requests.Response() resp.status_code = reply.status resp.headers = CaseInsensitiveDict(headers) - resp.raw = io.BytesIO(content) + resp.raw = _Raw(content, headers) resp._content = content resp.url = request.url resp.request = request diff --git a/simplyblock_core/storage_arrays/testing/dell_powerflex.py b/simplyblock_core/storage_arrays/testing/dell_powerflex.py new file mode 100644 index 0000000000..d1266297df --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/dell_powerflex.py @@ -0,0 +1,289 @@ +"""Fake Dell PowerFlex REST gateway (3.6 ``/api/login`` token auth, 4.x +``/rest/auth/login`` OAuth), the subset the adapter uses. + +Quirks reproduced: the 3.x login token is a JSON *string*; "not found" is +HTTP 500 with ``{"message", "httpStatusCode", "errorCode"}``; sizes round up +to 8 GiB; ``mappedSdcInfo`` is ``null`` when unmapped; a second SDC mapping +needs ``allowMultipleMappings``; SDCs exist before (and independently of) +any host registration. Error codes are placeholders, not PowerFlex's. +""" +from __future__ import annotations + +import json +import secrets +import uuid +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +GIB = 1 << 30 +_KB_PER_8GIB = 8 * 1024 * 1024 +_VOL = r"/api/instances/Volume::(?P[0-9a-zA-Z]+)" + + +def _err(msg: str, status: int = 500, code: int = 0) -> FakeError: + return FakeError(status, {"message": msg, "httpStatusCode": status, "errorCode": code}) + + +def _bool(v: Any) -> bool: + return str(v).upper() == "TRUE" + + +class FakePowerFlex(FakeArray): + vendor = "dell-powerflex" + + def __init__(self, username: str = "admin", password: str = "pf-pass", + generation: int = 4): + super().__init__() + if generation not in (3, 4): + raise ValueError("generation must be 3 or 4") + self.generation = generation + self.username = username + self.password = password + self.tokens: set[str] = set() # 3.x session tokens + self.access_tokens: set[str] = set() # 4.x bearer tokens + self.system_id = "4a54e5c8a1b2c3d4" + self.pools = {"7d2e1c5b00000001": {"id": "7d2e1c5b00000001", "name": "sb-pool", + "protectionDomainId": "e1f2a3b400000000"}} + self.volumes: dict[str, dict[str, Any]] = {} + self.sdcs: dict[str, dict[str, Any]] = {} + + # -- fake controls ------------------------------------------------------ + + @property + def pool_id(self) -> str: + return next(iter(self.pools)) + + def _new_id(self) -> str: + return f"{secrets.token_hex(4)}{self.next_id():08x}" + + def expire_sessions(self) -> None: + self.tokens.clear() + self.access_tokens.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields["password"] + + def add_sdc(self, guid: str, name: str | None = None) -> str: + sid = self._new_id() + self.sdcs[sid] = {"id": sid, "sdcGuid": guid, "name": name, + "sdcIp": f"10.0.0.{len(self.sdcs) + 10}", "sdcApproved": True, + "mdmConnectionState": "Connected", + "links": [{"rel": "self", "href": f"/api/instances/Sdc::{sid}"}]} + return sid + + def add_volume(self, name: str | None, size_bytes: int, pool_id: str | None = None) -> str: + vid = self._new_id() + kb = -(-max(size_bytes // 1024, 1) // _KB_PER_8GIB) * _KB_PER_8GIB + self.volumes[vid] = { + "id": vid, "name": name, "sizeInKb": kb, + "storagePoolId": pool_id or self.pool_id, "volumeType": "ThinProvisioned", + "vtreeId": self._new_id(), "ancestorVolumeId": None, "mappedSdcInfo": None, + "creationTime": 1763600000, "isObfuscated": False, + "links": [{"rel": "self", "href": f"/api/instances/Volume::{vid}"}], + } + return vid + + # -- auth --------------------------------------------------------------- + + def _auth(self, req: Request) -> None: + if self.generation == 3: + creds = req.basic_auth() + if creds and creds[1] in self.tokens: + return + else: + header = req.headers.get("Authorization", "") + if header.startswith("Bearer ") and header[7:] in self.access_tokens: + return + raise _err("Unauthorized", 401) + + @route("GET", "/api/login") + def api_login(self, req): + if self.generation != 3: + raise _err("Not found", 404) + if req.basic_auth() != (self.username, self.password): + raise _err("Unauthorized", 401) + token = secrets.token_urlsafe(24) + self.tokens.add(token) + # The 3.x gateway returns the token as a JSON string literal. + return Reply(200, json.dumps(token).encode(), {"Content-Type": "application/json"}) + + @route("GET", "/api/logout") + def api_logout(self, req): + creds = req.basic_auth() + if creds: + self.tokens.discard(creds[1]) + return Reply(200, None) + + @route("POST", "/rest/auth/login") + def rest_login(self, req): + if self.generation != 4: + return Reply(404, b"HTTP Status 404 - Not Found", + {"Content-Type": "text/html"}) + body = req.json() or {} + if (body.get("username"), body.get("password")) != (self.username, self.password): + raise FakeError(401, {"error": "invalid_grant", + "error_description": "Invalid user credentials"}) + token = secrets.token_urlsafe(32) + self.access_tokens.add(token) + return Reply(200, {"access_token": token, "expires_in": 300, + "refresh_token": secrets.token_urlsafe(32), + "refresh_expires_in": 1800, "token_type": "Bearer", + "scope": "openid profile email"}) + + @route("POST", "/rest/auth/logout") + def rest_logout(self, req): + header = req.headers.get("Authorization", "") + self.access_tokens.discard(header[7:]) + return Reply(204, None) + + # -- system ----------------------------------------------------------------- + + @route("GET", "/api/types/System/instances") + def systems(self, req): + self._auth(req) + version = "R3_6.0.0" if self.generation == 3 else "R4_5.0.0" + return Reply(200, [{"id": self.system_id, "name": "fake-pflex", + "systemVersionName": f"DellEMC PowerFlex Version: {version}", + "mdmClusterState": "ClusteredNormal"}]) + + # -- volumes ------------------------------------------------------------- + + def _vol(self, vid: str) -> dict[str, Any]: + if vid not in self.volumes: + raise _err("Could not find the volume", 500, 79) + return self.volumes[vid] + + @route("POST", "/api/types/Volume/instances") + def create_volume(self, req): + self._auth(req) + body = req.json() or {} + name = body.get("name") + if name and len(name) > 31: + raise _err("Name is too long", 500, 7) + if name and any(v["name"] == name for v in self.volumes.values()): + raise _err("Volume name already in use", 500, 6) + pool = body.get("storagePoolId") + if pool not in self.pools: + raise _err("Could not find the Storage Pool", 500, 61) + if body.get("volumeType") not in ("ThinProvisioned", "ThickProvisioned"): + raise _err("Invalid volume type", 400, 1) + try: + kb = int(body["volumeSizeInKb"]) + except (KeyError, TypeError, ValueError): + raise _err("Invalid volume size", 400, 1) from None + vid = self.add_volume(name, kb * 1024, pool) + return Reply(200, {"id": vid}) + + @route("GET", _VOL) + def get_volume(self, req, vid): + self._auth(req) + return Reply(200, self._vol(vid)) + + @route("POST", "/api/types/Volume/instances/action/queryIdByKey") + def query_id(self, req): + self._auth(req) + name = (req.json() or {}).get("name") + for v in self.volumes.values(): + if v["name"] == name: + return Reply(200, v["id"]) + raise _err("Could not find the volume", 500, 79) + + @route("POST", _VOL + "/action/(?P[A-Za-z]+)") + def volume_action(self, req, vid, action): + self._auth(req) + v = self._vol(vid) + body = req.json() or {} + mapped = v["mappedSdcInfo"] or [] + if action == "setVolumeSize": + gb = int(body["sizeInGB"]) + kb = -(-gb // 8) * 8 * 1024 * 1024 + if kb < v["sizeInKb"]: + raise _err("Volume size cannot be decreased", 500, 76) + v["sizeInKb"] = kb + return Reply(200, {}) + if action == "removeVolume": + if body.get("removeMode") not in ("ONLY_ME", "INCLUDING_DESCENDANTS", + "DESCENDANTS_ONLY", "WHOLE_VTREE"): + raise _err("Invalid remove mode", 400, 1) + if mapped: + raise _err("The volume is mapped to an SDC; unmap it first", 500, 80) + del self.volumes[vid] + return Reply(200, {}) + if action == "addMappedSdc": + sid = body.get("sdcId") + if sid not in self.sdcs: + raise _err("Could not find the SDC", 500, 100) + if any(m["sdcId"] == sid for m in mapped): + raise _err("The volume is already mapped to this SDC", 500, 81) + if mapped and not _bool(body.get("allowMultipleMappings")): + raise _err("The volume is already mapped to another SDC; " + "set allowMultipleMappings", 500, 82) + sdc = self.sdcs[sid] + v["mappedSdcInfo"] = mapped + [{ + "sdcId": sid, "sdcIp": sdc["sdcIp"], "sdcName": sdc["name"], + "limitIops": 0, "limitBwInMbps": 0, + "accessMode": body.get("accessMode", "ReadWrite")}] + return Reply(200, {}) + if action == "removeMappedSdc": + sid = body.get("sdcId") + rest = [m for m in mapped if m["sdcId"] != sid] + if len(rest) == len(mapped): + raise _err("The volume is not mapped to SDC", 500, 83) + v["mappedSdcInfo"] = rest or None + return Reply(200, {}) + raise _err(f"Unknown action {action}", 400, 1) + + # -- SDCs ----------------------------------------------------------------- + + @route("GET", "/api/types/Sdc/instances") + def list_sdcs(self, req): + self._auth(req) + return Reply(200, list(self.sdcs.values())) + + @route("POST", r"/api/instances/Sdc::(?P[0-9a-zA-Z]+)/action/setSdcName") + def set_sdc_name(self, req, sid): + self._auth(req) + if sid not in self.sdcs: + raise _err("Could not find the SDC", 500, 100) + name = (req.json() or {}).get("sdcName") + if any(s["name"] == name and s["id"] != sid for s in self.sdcs.values()): + raise _err("SDC name already in use", 500, 6) + self.sdcs[sid]["name"] = name + return Reply(200, {}) + + +def _guid() -> str: + return str(uuid.uuid4()).upper() + + +def contract_setup(generation: int = 4) -> ContractSetup: + fake = FakePowerFlex(username="admin", password="pf-pass", generation=generation) + guid_a, guid_b = _guid(), _guid() + # SDCs register themselves with the MDM when installed on a host; the + # adapter only finds them, so the fake starts with both hosts' SDCs. + fake.add_sdc(guid_a) + fake.add_sdc(guid_b) + return ContractSetup( + fake=fake, + config=make_config("dell-powerflex", "https://powerflex.fake.example", + storage_pool_id=fake.pool_id), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("pf-pass")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.SDC: ( + (Initiator(protocol=Protocol.SDC, address=guid_a),), + (Initiator(protocol=Protocol.SDC, address=guid_b),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + has_lun_numbers=False, + ) diff --git a/simplyblock_core/storage_arrays/testing/dell_powermax.py b/simplyblock_core/storage_arrays/testing/dell_powermax.py new file mode 100644 index 0000000000..5c3e59f406 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/dell_powermax.py @@ -0,0 +1,593 @@ +"""Fake Dell PowerMax Unisphere REST API 10.x (the subset the adapter uses). + +Quirks reproduced: basic auth on every request (401 plain-text body); +``{"message": "..."}`` error bodies, 400 for most failures and 404 for +missing objects; versioned paths (``/univmax/restapi/{ver}/...``) with +unknown versions refused; sizes in cylinders (1 CYL = 1920 KiB), also +accepting MB/GB/TB; ``executionOption: ASYNCHRONOUS`` answered with +``202`` + job that runs on the second poll of ``/{ver}/system/job/{id}`` +(``SCHEDULED`` -> ``RUNNING`` -> ``SUCCEEDED``/``FAILED`` with the error in +``result``); volume listing as an iterator (``count``/``resultList``, no +``resultList`` when empty); non-unique volume identifiers; masked storage +groups cannot lose their last device; host LUN addresses in hex; port groups +bound to one protocol. +""" +from __future__ import annotations + +import math +import re +import secrets +from collections.abc import Callable +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +SYMMETRIX_ID = "000197900123" +CYL_BYTES = 1920 * 1024 +_ROOT = "/univmax/restapi" +_V = _ROOT + r"/(?P\d+)" +_SLO = _V + r"/sloprovisioning/symmetrix/(?P\d{12})" +_NAME_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$") +_UNIT_MB = {"MB": 1, "GB": 1024, "TB": 1024 * 1024} + + +def _err(status: int, message: str) -> FakeError: + return FakeError(status, {"message": message}) + + +def _bad(message: str) -> FakeError: + return _err(400, message) + + +def _proto_of(initiator: str) -> str: + v = initiator.lower() + if v.startswith(("iqn.", "eui.")): + return "iscsi" + if v.startswith("nqn."): + return "nvme-tcp" + return "fc" + + +class FakeDellPowerMax(FakeArray): + vendor = "dell-powermax" + api_versions = ("100", "101") + + def __init__(self, username: str = "smc", password: str = "pmax-pass", + symmetrix_id: str = SYMMETRIX_ID): + super().__init__() + self.username = username + self.password = password + self.symmetrix_id = symmetrix_id + self.volumes: dict[str, dict[str, Any]] = {} # device id -> volume + self.storage_groups: dict[str, dict[str, Any]] = {} # name -> sg + self.hosts: dict[str, dict[str, Any]] = {} # name -> host + self.port_groups: dict[str, dict[str, Any]] = { + "sb_pg_iscsi": {"protocol": "iscsi", "ports": ["SE-1E:0", "SE-2E:0"]}, + "sb_pg_fc": {"protocol": "fc", "ports": ["FA-1D:4", "FA-2D:4"]}, + "sb_pg_nvme": {"protocol": "nvme-tcp", "ports": ["OR-1C:2", "OR-2C:2"]}, + } + self.views: dict[str, dict[str, Any]] = {} # name -> masking view + self.jobs: dict[str, dict[str, Any]] = {} + self._job_fns: dict[str, Callable[[], Reply]] = {} + self.first_lun = 1 + self._dev = 0x00100 + + # -- fake controls ------------------------------------------------------ + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields.get("password", self.password) + + def _new_device(self, identifier: str, cyl: int) -> str: + self._dev += 1 + dev = f"{self._dev:05X}" + wwn = ("6000097000" + self.symmetrix_id[-12:] + dev.encode().hex()).upper() + self.volumes[dev] = { + "volumeId": dev, "type": "TDEV", "emulation": "FBA", "ssid": "FFFFFFFF", + "status": "Ready", "reserved": False, "pinned": False, + "volume_identifier": identifier, "cap_cyl": cyl, "wwn": wwn, + "effective_wwn": wwn, "has_effective_wwn": False, "encapsulated": False, + "storageGroupId": [], "snapvx_source": False, "snapvx_target": False, + "mobility_id_enabled": False, + } + return dev + + def add_volume(self, name: str, size: int, storage_group: str | None = None) -> str: + """Create a device directly (optionally inside a storage group).""" + dev = self._new_device(name, math.ceil(size / CYL_BYTES)) + if storage_group: + sg = self.storage_groups.setdefault(storage_group, self._sg_record( + storage_group, "SRP_1", None)) + sg["volumes"].append(dev) + self.volumes[dev]["storageGroupId"].append(storage_group) + return dev + + # -- dispatch helpers ---------------------------------------------------------- + + def _auth(self, req: Request) -> None: + if req.basic_auth() != (self.username, self.password): + raise FakeError(401, "Unauthorized", {"WWW-Authenticate": 'Basic realm="Unisphere"'}) + + def _check(self, req: Request, ver: str, sym: str | None = None) -> None: + self._auth(req) + if ver not in self.api_versions: + raise _err(404, f"Unsupported REST API version {ver}") + if sym is not None and sym != self.symmetrix_id: + raise _err(404, f"Cannot find Symmetrix {sym}") + + def _run(self, req: Request, fn: Callable[[], Reply]) -> Reply: + """Honour ``executionOption``: run now, or queue as a job.""" + body = req.json() or {} + option = body.get("executionOption", "SYNCHRONOUS") + if option not in ("SYNCHRONOUS", "ASYNCHRONOUS"): + raise _bad(f"Invalid executionOption {option}") + if option == "SYNCHRONOUS": + return fn() + job_id = str(1_000_000 + self.next_id()) + self.jobs[job_id] = { + "jobId": job_id, "name": f"{req.method} {req.path}", "status": "SCHEDULED", + "username": self.username, "result": None, + "last_modified_date": "Nov-20-2026 00:00:00.000", + "resourceLink": f"https://unisphere.fake.example{req.path}", + "task": [{"execution_order": 1, "description": f"{req.method} {req.path}"}], + "_polls": 0, + } + self._job_fns[job_id] = fn + return Reply(202, self._public_job(job_id)) + + def _public_job(self, job_id: str) -> dict[str, Any]: + return {k: v for k, v in self.jobs[job_id].items() if not k.startswith("_")} + + @route("GET", _V + r"/system/job/(?P\d+)") + def get_job(self, req, ver, job_id): + self._check(req, ver) + job = self.jobs.get(job_id) + if job is None: + raise _err(404, f"Cannot find job {job_id}") + job["_polls"] += 1 + if job["status"] == "SCHEDULED": + job["status"] = "RUNNING" + elif job["status"] == "RUNNING": + fn = self._job_fns.pop(job_id) + try: + fn() + job["status"], job["result"] = "SUCCEEDED", "Succeeded" + except FakeError as e: + body = e.reply.body if isinstance(e.reply.body, dict) else {} + job["status"] = "FAILED" + job["result"] = f"Failed : {body.get('message', e.reply.status)}" + return Reply(200, self._public_job(job_id)) + + # -- system ------------------------------------------------------------------------- + + @route("GET", _ROOT + "/version") + def version(self, req): + self._auth(req) + return Reply(200, {"version": "T10.1.0.0", "api_version": "101", + "supported_api_versions": list(self.api_versions)}) + + @route("GET", _V + r"/system/symmetrix/(?P\d{12})") + def get_symmetrix(self, req, ver, sym): + self._check(req, ver, sym) + return Reply(200, {"symmetrixId": sym, "device_count": len(self.volumes), + "ucode": "6079.225.0", "model": "PowerMax_8500", + "local": True, "default_fba_srp": "SRP_1"}) + + # -- volumes ------------------------------------------------------------------------ + + def _vol(self, dev: str) -> dict[str, Any]: + if dev not in self.volumes: + raise _err(404, f"Cannot find Volume {dev}") + return self.volumes[dev] + + def _vol_view(self, v: dict[str, Any]) -> dict[str, Any]: + cyl = v["cap_cyl"] + mapped = any(v["volumeId"] in self.storage_groups[sg]["volumes"] + and self.storage_groups[sg]["maskingview"] for sg in v["storageGroupId"]) + return {**v, + "cap_gb": round(cyl * 1920 / 1024 / 1024, 2), + "cap_mb": round(cyl * 1.875, 2), + "num_of_storage_groups": len(v["storageGroupId"]), + "num_of_front_end_paths": 0, "mapped": mapped, + "allocated_percent": 0} + + @staticmethod + def _cylinders(attrs: dict[str, Any]) -> int: + unit = attrs.get("capacityUnit", "GB") + try: + size = float(attrs["volume_size"]) + except (KeyError, TypeError, ValueError) as e: + raise _bad("Invalid volume_size") from e + if size <= 0: + raise _bad("volume_size must be positive") + if unit == "CYL": + if size != int(size): + raise _bad("volume_size in CYL must be a whole number") + return int(size) + if unit not in _UNIT_MB: + raise _bad(f"Invalid capacityUnit {unit}") + return math.ceil(size * _UNIT_MB[unit] / 1.875) + + def _identifier(self, attrs: dict[str, Any]) -> str: + ident = attrs.get("volumeIdentifier") or {} + if not ident: + return "" + if ident.get("volumeIdentifierChoice") != "identifier_name": + raise _bad("Unsupported volumeIdentifierChoice") + name = ident.get("identifier_name") or "" + if len(name) > 64: + raise _bad("The volume identifier must not exceed 64 characters") + return name + + @route("GET", _SLO + "/volume") + def list_volumes(self, req, ver, sym): + self._check(req, ver, sym) + rows = list(self.volumes.values()) + if req.q("volume_identifier") is not None: + want = req.q("volume_identifier") or "" + if want.startswith(""): + rows = [v for v in rows if want[6:] in v["volume_identifier"]] + else: + rows = [v for v in rows if v["volume_identifier"] == want] + if req.q("storageGroupId") is not None: + sg = req.q("storageGroupId") + rows = [v for v in rows if sg in v["storageGroupId"]] + body: dict[str, Any] = {"count": len(rows), "maxPageSize": 1000, + "expirationTime": 1_900_000_000_000} + if rows: + body["id"] = f"{secrets.token_hex(4)}-iterator" + body["resultList"] = {"result": [{"volumeId": v["volumeId"]} for v in rows], + "from": 1, "to": len(rows)} + return Reply(200, body) + + @route("GET", _SLO + r"/volume/(?P[0-9A-Fa-f]+)") + def get_volume(self, req, ver, sym, dev): + self._check(req, ver, sym) + return Reply(200, self._vol_view(self._vol(dev))) + + @route("PUT", _SLO + r"/volume/(?P[0-9A-Fa-f]+)") + def put_volume(self, req, ver, sym, dev): + self._check(req, ver, sym) + vol = self._vol(dev) + body = req.json() or {} + expand = ((body.get("editVolumeActionParam") or {}).get("expandVolumeParam") or {}) + if not expand: + raise _bad("Unsupported editVolumeActionParam") + cyl = self._cylinders(expand.get("volumeAttribute") or {}) + + def do() -> Reply: + if cyl < vol["cap_cyl"]: + raise _bad(f"The new size of volume {dev} must be larger than the " + "current size") + vol["cap_cyl"] = cyl + return Reply(200, self._vol_view(vol)) + return self._run(req, do) + + @route("DELETE", _SLO + r"/volume/(?P[0-9A-Fa-f]+)") + def delete_volume(self, req, ver, sym, dev): + self._check(req, ver, sym) + vol = self._vol(dev) + if vol["storageGroupId"]: + raise _bad(f"A problem occurred deleting the volume {dev}: the device is a " + f"member of storage group {vol['storageGroupId'][0]}") + del self.volumes[dev] + return Reply(204) + + # -- storage groups ------------------------------------------------------------------ + + @staticmethod + def _sg_record(name: str, srp: str, slo: str | None) -> dict[str, Any]: + return {"storageGroupId": name, "srp": srp, "slo": slo or "NONE", + "volumes": [], "maskingview": []} + + def _sg(self, name: str) -> dict[str, Any]: + if name not in self.storage_groups: + raise _err(404, f"Cannot find Storage Group {name}") + return self.storage_groups[name] + + def _sg_view(self, sg: dict[str, Any]) -> dict[str, Any]: + cyl = sum(self.volumes[d]["cap_cyl"] for d in sg["volumes"]) + out = {"storageGroupId": sg["storageGroupId"], "srp": sg["srp"], "slo": sg["slo"], + "num_of_vols": len(sg["volumes"]), "num_of_child_sgs": 0, + "num_of_parent_sgs": 0, "num_of_masking_views": len(sg["maskingview"]), + "num_of_snapshots": 0, "cap_gb": round(cyl * 1920 / 1024 / 1024, 2), + "device_emulation": "FBA", "type": "Standalone", "unprotected": True, + "compression": True} + if sg["maskingview"]: + out["maskingview"] = list(sg["maskingview"]) + return out + + def _add_devices(self, sg: dict[str, Any], attrs_list: list[dict[str, Any]]) -> None: + for attrs in attrs_list: + cyl = self._cylinders(attrs) + ident = self._identifier(attrs) + for _ in range(int(attrs.get("num_of_vols", 1))): + dev = self._new_device(ident, cyl) + sg["volumes"].append(dev) + self.volumes[dev]["storageGroupId"].append(sg["storageGroupId"]) + + @route("POST", _SLO + "/storagegroup") + def post_sg(self, req, ver, sym): + self._check(req, ver, sym) + body = req.json() or {} + name = body.get("storageGroupId") or "" + if not _NAME_RE.match(name): + raise _bad(f"Invalid storage group name {name!r}") + srp = body.get("srpId", "None") + if srp not in ("SRP_1", "None"): + raise _err(404, f"Cannot find SRP {srp}") + params = body.get("sloBasedStorageGroupParam") or [] + for p in params: + if p.get("sloId", "None") not in ("None", "Diamond", "Platinum", "Gold", + "Silver", "Bronze", "Optimized"): + raise _bad(f"Invalid sloId {p.get('sloId')}") + if "workloadSelection" in p and p["workloadSelection"] not in ("None", None): + raise _bad("workloadSelection is not supported on this array") + + def do() -> Reply: + if name in self.storage_groups: + raise _bad(f"Storage Group {name} already exists") + slo = next((p.get("sloId") for p in params if p.get("sloId") != "None"), None) + sg = self._sg_record(name, srp, slo) + self.storage_groups[name] = sg + try: + for p in params: + self._add_devices(sg, p.get("volumeAttributes") or []) + except FakeError: + del self.storage_groups[name] + raise + return Reply(200, self._sg_view(sg)) + return self._run(req, do) + + @route("GET", _SLO + r"/storagegroup/(?P[^/]+)") + def get_sg(self, req, ver, sym, name): + self._check(req, ver, sym) + return Reply(200, self._sg_view(self._sg(name))) + + @route("PUT", _SLO + r"/storagegroup/(?P[^/]+)") + def put_sg(self, req, ver, sym, name): + self._check(req, ver, sym) + sg = self._sg(name) + edit = (req.json() or {}).get("editStorageGroupActionParam") or {} + if "removeVolumeParam" in edit: + devs = edit["removeVolumeParam"].get("volumeId") or [] + + def remove() -> Reply: + for d in devs: + if d not in sg["volumes"]: + raise _bad(f"Volume {d} is not in Storage Group {name}") + if sg["maskingview"] and len(set(sg["volumes"]) - set(devs)) == 0: + raise _bad(f"Cannot remove the last volume from Storage Group {name} " + "because it is part of a masking view") + for d in devs: + sg["volumes"].remove(d) + self.volumes[d]["storageGroupId"].remove(name) + return Reply(200, self._sg_view(sg)) + return self._run(req, remove) + expand = edit.get("expandStorageGroupParam") or {} + add = expand.get("addVolumeParam") + if add is not None: + if not add.get("create_new_volumes", False): + raise _bad("Only create_new_volumes is supported by the fake") + + def grow() -> Reply: + self._add_devices(sg, add.get("volumeAttributes") or []) + return Reply(200, self._sg_view(sg)) + return self._run(req, grow) + raise _bad("Unsupported editStorageGroupActionParam") + + @route("DELETE", _SLO + r"/storagegroup/(?P[^/]+)") + def delete_sg(self, req, ver, sym, name): + self._check(req, ver, sym) + sg = self._sg(name) + if sg["maskingview"]: + raise _bad(f"Storage Group {name} is part of masking view {sg['maskingview'][0]}") + if sg["volumes"]: + raise _bad(f"Storage Group {name} is not empty") + del self.storage_groups[name] + return Reply(204) + + # -- hosts ------------------------------------------------------------------------------- + + def _host(self, name: str) -> dict[str, Any]: + if name not in self.hosts: + raise _err(404, f"Cannot find Host {name}") + return self.hosts[name] + + def _host_view(self, h: dict[str, Any]) -> dict[str, Any]: + return {"hostId": h["hostId"], "num_of_masking_views": len(h["maskingview"]), + "num_of_initiators": len(h["initiator"]), "num_of_host_groups": 0, + "port_flags_override": False, "consistent_lun": False, + "initiator": list(h["initiator"]), "maskingview": list(h["maskingview"]), + "type": {"fc": "Fibre", "iscsi": "iSCSI", "nvme-tcp": "NVMe/TCP"}.get( + _proto_of(h["initiator"][0]) if h["initiator"] else "", "")} + + def _claim(self, host: str, initiators: list[str]) -> list[str]: + out = [] + for i in initiators: + proto = _proto_of(i) + if proto == "fc" and not re.match(r"^[0-9a-f]{16}$", i): + raise _bad(f"Invalid initiator {i}: expected a 16-digit WWN") + for other in self.hosts.values(): + if i.lower() in (x.lower() for x in other["initiator"]): + if other["hostId"] == host: + raise _bad(f"Initiator {i} is already in Host {host}") + raise _bad(f"Initiator {i} is already in use by Host {other['hostId']}") + out.append(i) + protos = {_proto_of(x) for x in out + (self.hosts[host]["initiator"] + if host in self.hosts else [])} + if len(protos) > 1: + raise _bad("A host cannot mix initiator types") + return out + + @route("POST", _SLO + "/host") + def post_host(self, req, ver, sym): + self._check(req, ver, sym) + body = req.json() or {} + name = body.get("hostId") or "" + if not _NAME_RE.match(name): + raise _bad(f"Invalid host name {name!r}") + initiators = body.get("initiatorId") or [] + + def do() -> Reply: + if name in self.hosts: + raise _bad(f"Host {name} already exists") + claimed = self._claim(name, initiators) + self.hosts[name] = {"hostId": name, "initiator": claimed, "maskingview": []} + return Reply(200, self._host_view(self.hosts[name])) + return self._run(req, do) + + @route("GET", _SLO + r"/host/(?P[^/]+)") + def get_host(self, req, ver, sym, name): + self._check(req, ver, sym) + return Reply(200, self._host_view(self._host(name))) + + @route("PUT", _SLO + r"/host/(?P[^/]+)") + def put_host(self, req, ver, sym, name): + self._check(req, ver, sym) + host = self._host(name) + edit = (req.json() or {}).get("editHostActionParam") or {} + add = (edit.get("addInitiatorParam") or {}).get("initiator") + if add is None: + raise _bad("Unsupported editHostActionParam") + + def do() -> Reply: + host["initiator"].extend(self._claim(name, add)) + return Reply(200, self._host_view(host)) + return self._run(req, do) + + @route("DELETE", _SLO + r"/host/(?P[^/]+)") + def delete_host(self, req, ver, sym, name): + self._check(req, ver, sym) + host = self._host(name) + if host["maskingview"]: + raise _bad(f"Host {name} is part of masking view {host['maskingview'][0]}") + del self.hosts[name] + return Reply(204) + + # -- masking views --------------------------------------------------------------- + + def _view(self, name: str) -> dict[str, Any]: + if name not in self.views: + raise _err(404, f"Cannot find Masking View {name}") + return self.views[name] + + @route("POST", _SLO + "/maskingview") + def post_view(self, req, ver, sym): + self._check(req, ver, sym) + body = req.json() or {} + name = body.get("maskingViewId") or "" + if not _NAME_RE.match(name): + raise _bad(f"Invalid masking view name {name!r}") + host = ((body.get("hostOrHostGroupSelection") or {}) + .get("useExistingHostParam") or {}).get("hostId") + pg = ((body.get("portGroupSelection") or {}) + .get("useExistingPortGroupParam") or {}).get("portGroupId") + sg = ((body.get("storageGroupSelection") or {}) + .get("useExistingStorageGroupParam") or {}).get("storageGroupId") + if not (host and pg and sg): + raise _bad("A masking view needs an existing host, port group and storage group") + + def do() -> Reply: + if name in self.views: + raise _bad(f"Masking View {name} already exists") + h, s = self._host(host), self._sg(sg) + if pg not in self.port_groups: + raise _err(404, f"Cannot find Port Group {pg}") + if not s["volumes"]: + raise _bad(f"Storage Group {sg} is empty") + protos = {_proto_of(i) for i in h["initiator"]} + if protos and protos != {self.port_groups[pg]["protocol"]}: + raise _bad(f"Port Group {pg} does not match the protocol of Host {host}") + used = {lun for v in self.views.values() if v["host"] == host + for lun in v["luns"].values()} + luns = {} + for dev in s["volumes"]: + lun = next(n for n in range(self.first_lun, 4096) if n not in used) + used.add(lun) + luns[dev] = lun + self.views[name] = {"maskingViewId": name, "host": host, "pg": pg, "sg": sg, + "luns": luns} + h["maskingview"].append(name) + s["maskingview"].append(name) + return Reply(200, self._view_view(self.views[name])) + return self._run(req, do) + + @staticmethod + def _view_view(v: dict[str, Any]) -> dict[str, Any]: + return {"maskingViewId": v["maskingViewId"], "hostId": v["host"], + "portGroupId": v["pg"], "storageGroupId": v["sg"]} + + @route("GET", _SLO + r"/maskingview/(?P[^/]+)") + def get_view(self, req, ver, sym, name): + self._check(req, ver, sym) + return Reply(200, self._view_view(self._view(name))) + + @route("GET", _SLO + r"/maskingview/(?P[^/]+)/connections") + def get_connections(self, req, ver, sym, name): + self._check(req, ver, sym) + v = self._view(name) + want = req.q("volume_id") + rows = [] + for dev in self.storage_groups[v["sg"]]["volumes"]: + if want and dev != want: + continue + lun = v["luns"].get(dev) + if lun is None: # device added after the view was created + continue + for initiator in self.hosts[v["host"]]["initiator"]: + for port in self.port_groups[v["pg"]]["ports"]: + rows.append({"volumeId": dev, "host_lun_address": f"{lun:04X}", + "cap_gb": round(self.volumes[dev]["cap_cyl"] * 1920 + / 1024 / 1024, 2), + "initiatorId": initiator, "alias": initiator, + "dir_port": port, "logged_in": False, "on_fabric": False}) + return Reply(200, {"maskingViewConnection": rows}) + + @route("DELETE", _SLO + r"/maskingview/(?P[^/]+)") + def delete_view(self, req, ver, sym, name): + self._check(req, ver, sym) + v = self._view(name) + self.hosts[v["host"]]["maskingview"].remove(name) + self.storage_groups[v["sg"]]["maskingview"].remove(name) + del self.views[name] + return Reply(204) + + +def contract_setup(**options: Any) -> ContractSetup: + fake = FakeDellPowerMax(username="smc", password="pmax-pass") + opts: dict[str, Any] = { + "symmetrix_id": SYMMETRIX_ID, + "port_group": {"iscsi": "sb_pg_iscsi", "fc": "sb_pg_fc", "nvme-tcp": "sb_pg_nvme"}, + } + opts.update(options) + return ContractSetup( + fake=fake, + config=make_config("dell-powermax", "https://unisphere.fake.example:8443", **opts), + credential=Credential({"username": SecretStr("smc"), + "password": SecretStr("pmax-pass")}), + bad_credential=Credential({"username": SecretStr("smc"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + Protocol.NVME_TCP: ( + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-a"),), + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-b"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size, "customer_sg"), + ) diff --git a/simplyblock_core/storage_arrays/testing/dell_powerstore.py b/simplyblock_core/storage_arrays/testing/dell_powerstore.py new file mode 100644 index 0000000000..0855735314 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/dell_powerstore.py @@ -0,0 +1,426 @@ +"""Fake Dell PowerStore REST API (the subset the adapter uses). + +Quirks reproduced: basic auth only on ``login_session``; session cookie +``auth_cookie`` plus ``DELL-EMC-TOKEN`` CSRF header required on non-GET +requests (403 without it); PostgREST-style ``select``/``eq.`` filters with +400 on unknown fields; ``limit``/``offset`` paging answered with 206 and +``Content-Range``; 201 ``{"id"}`` on create, 204 on modify/delete/attach; +422 with ``{"messages": [...]}`` for semantic errors; sizes in bytes, a +multiple of 8192; ``wwn``/``nguid`` prefixed ``naa.``/``nguid.``. +""" +from __future__ import annotations + +import re +import secrets +import uuid +from http.cookies import CookieError, SimpleCookie +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +API = "/api/rest" +_ID = r"(?P[^/]+)" +_FC_RE = re.compile(r"^([0-9a-f]{2}:){7}[0-9a-f]{2}$", re.IGNORECASE) +MIN_SIZE = 1 << 20 +MAX_SIZE = 256 << 40 +GRANULARITY = 8192 + +_FIELDS = { + "volume": {"id", "name", "description", "type", "wwn", "nguid", "size", "state", + "appliance_id", "creation_timestamp", "is_replication_destination", + "performance_policy_id", "protection_policy_id"}, + "host": {"id", "name", "description", "os_type", "host_group_id", "host_initiators", + "host_connectivity"}, + "host_volume_mapping": {"id", "host_id", "host_group_id", "volume_id", + "logical_unit_number"}, + "cluster": {"id", "global_id", "name", "management_address", "state", + "appliance_count"}, + "appliance": {"id", "name", "model", "service_tag", "express_service_code"}, + "software_installed": {"id", "release_version", "build_version", "is_cluster", + "release_timestamp"}, +} + + +def _err(status: int, message: str, code: str = "0xE04040010005") -> FakeError: + return FakeError(status, {"messages": [{"code": code, "severity": "Error", + "message_l10n": message, "arguments": []}]}) + + +def _not_found(kind: str, oid: str) -> FakeError: + return _err(404, f"The {kind} {oid} was not found.", "0xE04040010005") + + +def _unprocessable(message: str) -> FakeError: + return _err(422, message, "0xE0A07001000C") + + +class FakeDellPowerStore(FakeArray): + vendor = "dell-powerstore" + + def __init__(self, username: str = "admin", password: str = "ps-pass"): + super().__init__() + self.username = username + self.password = password + self.sessions: dict[str, str] = {} # auth_cookie -> CSRF token + self.volumes: dict[str, dict[str, Any]] = {} # id -> volume + self.hosts: dict[str, dict[str, Any]] = {} # id -> host + self.mappings: list[dict[str, Any]] = [] + self.page_limit = 2000 # array-side default/max page size + self.first_lun = 1 + + # -- fake controls ------------------------------------------------------ + + def expire_sessions(self) -> None: + self.sessions.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields.get("password", self.password) + + def add_volume(self, name: str, size: int) -> str: + vid = str(uuid.uuid4()) + self.volumes[vid] = { + "id": vid, "name": name, "description": None, "type": "Primary", + "wwn": "naa.68ccf098" + secrets.token_hex(12), + "nguid": "nguid." + secrets.token_hex(16), + "size": size, "state": "Ready", "appliance_id": "A1", + "creation_timestamp": "2026-11-20T00:00:00+00:00", + "is_replication_destination": False, "performance_policy_id": "default_medium", + "protection_policy_id": None, + } + return vid + + # -- auth --------------------------------------------------------------- + + def _session(self, req: Request) -> None: + cookie: str | None = None + try: + jar = SimpleCookie() + jar.load(req.headers.get("Cookie", "")) + if "auth_cookie" in jar: + cookie = jar["auth_cookie"].value + except CookieError: + pass + if cookie is None or cookie not in self.sessions: + raise _err(401, "Unauthorized: the session is invalid or expired.", + "0xE09040010001") + if req.method != "GET" and req.headers.get("DELL-EMC-TOKEN") != self.sessions[cookie]: + raise _err(403, "Forbidden: missing or invalid DELL-EMC-TOKEN.", "0xE09040010003") + + @route("GET", API + "/login_session") + def login_session(self, req): + if req.basic_auth() != (self.username, self.password): + raise _err(401, "Unauthorized: invalid credentials.", "0xE09040010001") + cookie, token = secrets.token_hex(16), secrets.token_hex(16) + self.sessions[cookie] = token + return Reply(200, [{"id": secrets.token_hex(8), "user": self.username, + "role_ids": ["1"], "idle_timeout": 3600, + "is_password_change_required": False, + "is_built_in_user": True}], + {"DELL-EMC-TOKEN": token, + "Set-Cookie": f"auth_cookie={cookie}; Path=/; Secure; HttpOnly"}) + + @route("POST", API + "/logout") + def logout(self, req): + self._session(req) + jar = SimpleCookie() + jar.load(req.headers.get("Cookie", "")) + self.sessions.pop(jar["auth_cookie"].value, None) + return Reply(204) + + # -- query engine ------------------------------------------------------------ + + def _query(self, req: Request, kind: str, rows: list[dict[str, Any]]) -> Reply: + fields = _FIELDS[kind] + for key, values in req.query.items(): + if key in ("select", "limit", "offset", "order"): + continue + if key not in fields: + raise _err(400, f"Invalid filter field: {key}.", "0xE04040020009") + m = re.match(r"^eq\.(.*)$", values[0], re.DOTALL) + if not m: + raise _err(400, f"Unsupported filter operator in {key}={values[0]}.", + "0xE04040020009") + want = m.group(1) + rows = [r for r in rows if _as_filter(r.get(key)) == want] + select = req.q("select") + if select: + names = select.split(",") + bad = [n for n in names if n not in fields] + if bad: + raise _err(400, f"Invalid select field: {','.join(bad)}.", "0xE04040020009") + else: + names = ["id"] + offset = int(req.q("offset", "0") or 0) + limit = min(int(req.q("limit", str(self.page_limit)) or self.page_limit), + self.page_limit) + total = len(rows) + page = rows[offset:offset + limit] + body = [{n: r.get(n) for n in names} for r in page] + if offset + len(page) < total: + end = offset + len(page) - 1 + return Reply(206, body, {"Content-Range": f"{offset}-{end}/{total}"}) + return Reply(200, body) + + def _project(self, req: Request, kind: str, row: dict[str, Any]) -> Reply: + select = req.q("select") + names = select.split(",") if select else sorted(_FIELDS[kind]) + bad = [n for n in names if n not in _FIELDS[kind]] + if bad: + raise _err(400, f"Invalid select field: {','.join(bad)}.", "0xE04040020009") + return Reply(200, {n: row.get(n) for n in names}) + + # -- system ------------------------------------------------------------------ + + @route("GET", API + "/cluster") + def get_cluster(self, req): + self._session(req) + return self._query(req, "cluster", [{ + "id": "0", "global_id": "PS4a7b5c3e9f01", "name": "fake-powerstore", + "management_address": "10.0.0.10", "state": "Configured", "appliance_count": 1}]) + + @route("GET", API + "/appliance") + def get_appliance(self, req): + self._session(req) + return self._query(req, "appliance", [{ + "id": "A1", "name": "fake-powerstore-appliance-1", "model": "PowerStore 1200T", + "service_tag": "FAKE123", "express_service_code": "0"}]) + + @route("GET", API + "/software_installed") + def get_software(self, req): + self._session(req) + return self._query(req, "software_installed", [ + {"id": "c1", "release_version": "4.0.0.0", "build_version": "4.0.0.0.5.123", + "is_cluster": True, "release_timestamp": "2026-01-01T00:00:00Z"}, + {"id": "a1", "release_version": "4.0.0.0", "build_version": "4.0.0.0.5.123", + "is_cluster": False, "release_timestamp": "2026-01-01T00:00:00Z"}, + ]) + + # -- volumes ----------------------------------------------------------------- + + def _vol(self, oid: str) -> dict[str, Any]: + if oid not in self.volumes: + raise _not_found("volume", oid) + return self.volumes[oid] + + @staticmethod + def _check_size(size: Any) -> int: + if not isinstance(size, int) or isinstance(size, bool): + raise _err(400, "Invalid value for size.", "0xE04040020009") + if size < MIN_SIZE or size > MAX_SIZE: + raise _err(400, f"Size {size} is out of range ({MIN_SIZE}..{MAX_SIZE}).", + "0xE04040020009") + if size % GRANULARITY: + raise _err(400, f"Size {size} must be a multiple of {GRANULARITY}.", + "0xE04040020009") + return size + + @route("GET", API + "/volume") + def list_volumes(self, req): + self._session(req) + return self._query(req, "volume", list(self.volumes.values())) + + @route("GET", API + "/volume/" + _ID) + def get_volume(self, req, oid): + self._session(req) + return self._project(req, "volume", self._vol(oid)) + + @route("POST", API + "/volume") + def post_volume(self, req): + self._session(req) + body = req.json() or {} + name = body.get("name") + if not name or len(name) > 128: + raise _err(400, "Invalid value for name.", "0xE04040020009") + size = self._check_size(body.get("size")) + if any(v["name"] == name for v in self.volumes.values()): + raise _unprocessable(f"The name {name} is already in use by another volume.") + vid = self.add_volume(name, size) + if body.get("appliance_id"): + self.volumes[vid]["appliance_id"] = body["appliance_id"] + return Reply(201, {"id": vid}) + + @route("PATCH", API + "/volume/" + _ID) + def patch_volume(self, req, oid): + self._session(req) + vol = self._vol(oid) + body = req.json() or {} + if "size" in body: + size = self._check_size(body["size"]) + if size < vol["size"]: + raise _unprocessable("The new size must be greater than the current size " + "of the volume.") + vol["size"] = size + return Reply(204) + + @route("DELETE", API + "/volume/" + _ID) + def delete_volume(self, req, oid): + self._session(req) + self._vol(oid) + if any(m["volume_id"] == oid for m in self.mappings): + raise _unprocessable(f"Volume {oid} cannot be deleted because it is attached " + "to one or more hosts.") + del self.volumes[oid] + return Reply(204) + + @route("POST", API + "/volume/" + _ID + "/attach") + def attach(self, req, oid): + self._session(req) + self._vol(oid) + body = req.json() or {} + host_id = body.get("host_id") + if not host_id or host_id not in self.hosts: + raise _not_found("host", str(host_id)) + if any(m["volume_id"] == oid and m["host_id"] == host_id for m in self.mappings): + raise _unprocessable(f"Volume {oid} is already attached to host {host_id}.") + used = {m["logical_unit_number"] for m in self.mappings if m["host_id"] == host_id} + lun = body.get("logical_unit_number") + if lun is None: + lun = next(n for n in range(self.first_lun, 16384) if n not in used) + elif lun in used: + raise _unprocessable(f"Logical unit number {lun} is already in use on the host.") + self.mappings.append({"id": str(uuid.uuid4()), "host_id": host_id, + "host_group_id": None, "volume_id": oid, + "logical_unit_number": lun}) + return Reply(204) + + @route("POST", API + "/volume/" + _ID + "/detach") + def detach(self, req, oid): + self._session(req) + self._vol(oid) + host_id = (req.json() or {}).get("host_id") + before = len(self.mappings) + self.mappings = [m for m in self.mappings + if not (m["volume_id"] == oid and m["host_id"] == host_id)] + if len(self.mappings) == before: + raise _unprocessable(f"Volume {oid} is not attached to host {host_id}.") + return Reply(204) + + @route("GET", API + "/host_volume_mapping") + def list_mappings(self, req): + self._session(req) + return self._query(req, "host_volume_mapping", list(self.mappings)) + + # -- hosts ---------------------------------------------------------------------- + + def _host(self, oid: str) -> dict[str, Any]: + if oid not in self.hosts: + raise _not_found("host", oid) + return self.hosts[oid] + + def _claim(self, host_id: str, initiators: Any) -> list[dict[str, Any]]: + if not isinstance(initiators, list): + raise _err(400, "Invalid value for initiators.", "0xE04040020009") + out = [] + for i in initiators: + port_name, port_type = i.get("port_name"), i.get("port_type") + if port_type not in ("iSCSI", "FC", "NVMe"): + raise _err(400, f"Invalid port_type {port_type}.", "0xE04040020009") + if not port_name: + raise _err(400, "Invalid value for port_name.", "0xE04040020009") + if port_type == "FC" and not _FC_RE.match(port_name): + raise _err(400, f"Invalid FC port_name {port_name}.", "0xE04040020009") + for other in self.hosts.values(): + for p in other["host_initiators"]: + if p["port_name"].lower() == port_name.lower(): + if other["id"] == host_id: + raise _unprocessable( + f"The initiator {port_name} is already added to this host.") + raise _unprocessable( + f"The initiator {port_name} is already registered to host " + f"{other['name']}.") + out.append({"port_name": port_name, "port_type": port_type, + "active_sessions": [], "chap_single_username": None, + "chap_mutual_username": None}) + return out + + @route("GET", API + "/host") + def list_hosts(self, req): + self._session(req) + return self._query(req, "host", list(self.hosts.values())) + + @route("GET", API + "/host/" + _ID) + def get_host(self, req, oid): + self._session(req) + return self._project(req, "host", self._host(oid)) + + @route("POST", API + "/host") + def post_host(self, req): + self._session(req) + body = req.json() or {} + name = body.get("name") + if not name or len(name) > 128: + raise _err(400, "Invalid value for name.", "0xE04040020009") + if body.get("os_type") not in ("Windows", "Linux", "ESXi", "AIX", "HP-UX", "Solaris"): + raise _err(400, f"Invalid os_type {body.get('os_type')}.", "0xE04040020009") + if any(h["name"] == name for h in self.hosts.values()): + raise _unprocessable(f"The name {name} is already in use by another host.") + hid = str(uuid.uuid4()) + initiators = self._claim(hid, body.get("initiators") or []) + self.hosts[hid] = {"id": hid, "name": name, "description": None, + "os_type": body["os_type"], "host_group_id": None, + "host_initiators": initiators, + "host_connectivity": "Local_Only"} + return Reply(201, {"id": hid}) + + @route("PATCH", API + "/host/" + _ID) + def patch_host(self, req, oid): + self._session(req) + host = self._host(oid) + body = req.json() or {} + if "initiators" in body: + raise _err(400, "Use add_initiators/remove_initiators to modify initiators.", + "0xE04040020009") + added = self._claim(oid, body.get("add_initiators") or []) + host["host_initiators"].extend(added) + return Reply(204) + + @route("DELETE", API + "/host/" + _ID) + def delete_host(self, req, oid): + self._session(req) + self._host(oid) + if any(m["host_id"] == oid for m in self.mappings): + raise _unprocessable(f"Host {oid} cannot be deleted because it has volumes " + "attached.") + del self.hosts[oid] + return Reply(204) + + +def _as_filter(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + return "" if value is None else str(value) + + +def contract_setup() -> ContractSetup: + fake = FakeDellPowerStore(username="admin", password="ps-pass") + return ContractSetup( + fake=fake, + config=make_config("dell-powerstore", "https://powerstore.fake.example"), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("ps-pass")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:FF:4C:1A:2C"),), + ), + Protocol.NVME_TCP: ( + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-a"),), + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-b"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/hitachi_vsp.py b/simplyblock_core/storage_arrays/testing/hitachi_vsp.py new file mode 100644 index 0000000000..cc09025c60 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/hitachi_vsp.py @@ -0,0 +1,616 @@ +"""Fake Hitachi VSP Configuration Manager REST API (the subset the adapter uses). + +Reproduces: both path layouts (embedded ``/ConfigurationManager/v1/objects`` +and server ``.../objects/storages/{storageDeviceId}``), basic-auth sessions +with ``Authorization: Session ``, ``202`` + job objects for every +change with the change applied only when the job runs (after ``job_polls`` +"Running" polls), job failures carrying the Hitachi error body, LDEVs that +read back as ``NOT DEFINED`` after deletion, per-port host groups with +default group 0, FC WWN uniqueness per port (iSCSI names may repeat across +targets, as on the array), LU paths listed on the LDEV, the resource-group +lock, and ``{"errorSource", "message", "solution", "messageId"}`` errors. +""" +from __future__ import annotations + +import re +import secrets +from collections.abc import Callable +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_ROOT = "/ConfigurationManager/v1/objects" +_P = _ROOT + r"(?:/storages/(?P\d{12}))?" +_MAX_LDEV = 65279 +NOT_DEFINED = "NOT DEFINED" +_UNITS = {"K": 1 << 10, "M": 1 << 20, "G": 1 << 30, "T": 1 << 40} + + +def _body(msg: str, msg_id: str, source: str = "", solution: str = "") -> dict[str, Any]: + return {"errorSource": source, "message": msg, "solution": solution or + "Check the request and the storage system configuration.", + "messageId": msg_id} + + +def _err(status: int, msg: str, msg_id: str = "KART40040-E", source: str = "") -> FakeError: + return FakeError(status, _body(msg, msg_id, source)) + + +def _storage_err(msg: str, ssb1: str = "2E11", ssb2: str = "0001") -> FakeError: + """A failure inside the storage system: in a job's ``error``.""" + body = _body(msg, "KART30000-E") + body["errorCode"] = {"SSB1": ssb1, "SSB2": ssb2} + return FakeError(400, body) + + +class FakeHitachiVsp(FakeArray): + vendor = "hitachi-vsp" + + def __init__(self, username: str = "sbadmin", password: str = "vsp-secret", + serial: int = 451234, model: str = "VSP E990", + ports: dict[str, str] | None = None, pools: tuple[int, ...] = (0,)): + super().__init__() + self.username, self.password = username, password + self.serial = serial + self.model = model + self.storage_device_id = f"{938000000000 + serial}" + self.sessions: dict[str, int] = {} # token -> sessionId + self.pools = set(pools) + self.ldevs: dict[int, dict[str, Any]] = {} # defined LDEVs only + self.ports = ports or {"CL1-A": "FIBRE", "CL2-A": "FIBRE", + "CL1-B": "ISCSI", "CL2-B": "ISCSI"} + #: (port, number) -> {"name", "mode", "wwns": [], "iscsis": [], "luns": {lun: ldev}} + self.groups: dict[tuple[str, int], dict[str, Any]] = {} + for port in self.ports: + self.groups[(port, 0)] = self._group(f"{port[2:].replace('-', '')}-G00", + "LINUX/IRIX") + self.jobs: dict[int, dict[str, Any]] = {} + self._pending: dict[int, Callable[[], list[str]]] = {} + self._polls_left: dict[int, int] = {} + #: "Running" answers before a job executes. + self.job_polls = 1 + #: Model requires the resource-group lock for changes. + self.require_lock = False + self.lock_owner: str | None = None + self.max_count = 16384 + + @staticmethod + def _group(name: str, mode: str) -> dict[str, Any]: + return {"name": name, "mode": mode, "modeOptions": [], "wwns": [], "iscsis": [], + "luns": {}} + + # -- fake controls ------------------------------------------------------ + + def expire_sessions(self) -> None: + self.sessions.clear() + self.lock_owner = None + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields["password"] + + def add_ldev(self, label: str, size: int, ldev_id: int | None = None) -> str: + lid = ldev_id if ldev_id is not None else self._free_ldev(0, _MAX_LDEV) + self.ldevs[lid] = {"pool": min(self.pools), "blocks": size // 512, "label": label} + return str(lid) + + def _free_ldev(self, start: int, end: int) -> int: + lid = next((i for i in range(start, end + 1) if i not in self.ldevs), None) + if lid is None: + raise _storage_err("No free LDEV number in the specified range.", "2E11", "0012") + return lid + + def naa(self, lid: int) -> str: + s = f"{self.serial:06X}" + return f"60060E80{s}0030{s}0000{lid:04X}" + + # -- auth / prefix ------------------------------------------------------- + + def _prefix(self, sdid: str | None) -> str: + if sdid is None: + return _ROOT + if sdid != self.storage_device_id: + raise _err(404, f"The storage system {sdid} is not registered.", "KART40053-E") + return f"{_ROOT}/storages/{sdid}" + + def _token(self, req: Request) -> str: + header = req.headers.get("Authorization", "") + token = header[8:] if header.startswith("Session ") else "" + if token not in self.sessions: + raise _err(401, "The session is invalid or has timed out.", "KART40047-E", + req.path) + return token + + def _auth(self, req: Request, sdid: str | None) -> str: + self._prefix(sdid) + return self._token(req) + + @route("POST", _P + "/sessions") + def post_session(self, req, sdid): + self._prefix(sdid) + if req.basic_auth() != (self.username, self.password): + raise _err(401, "Authentication failed.", "KART40046-E", req.path) + token = secrets.token_hex(16) + self.sessions[token] = self.next_id() + return Reply(200, {"token": token, "sessionId": self.sessions[token]}) + + @route("DELETE", _P + r"/sessions/(?P\d+)") + def delete_session(self, req, sdid, sid): + token = self._auth(req, sdid) + if self.sessions.get(token) != int(sid): + raise _err(404, f"Session {sid} does not exist.", "KART40050-E") + del self.sessions[token] + if self.lock_owner == token: + self.lock_owner = None + return Reply(200, {"sessionId": int(sid)}) + + @route("GET", _ROOT + r"/storages/(?Pinstance|\d{12})") + def get_storage(self, req, which): + self._auth(req, None if which == "instance" else which) + return Reply(200, {"storageDeviceId": self.storage_device_id, "model": self.model, + "serialNumber": self.serial, "svpIp": "192.0.2.10", + "dkcMicroVersion": "93-07-23-80/00", "communicationModes": [ + {"communicationMode": "lanConnectionMode"}]}) + + # -- jobs ----------------------------------------------------------------- + + def _job(self, req: Request, sdid: str | None, token: str, + execute: Callable[[], list[str]], *, lock_request: bool = False) -> Reply: + if not lock_request: + if self.lock_owner and self.lock_owner != token: + raise _err(409, "The resource group is locked by another user.", + "KART40049-E") + if self.require_lock and self.lock_owner != token: + raise _storage_err("The resource group is not locked.", "2E10", "A001") + jid = self.next_id() + prefix = self._prefix(sdid) + self.jobs[jid] = { + "jobId": jid, "self": f"{prefix}/jobs/{jid}", "userId": self.username, + "status": "Initializing", "state": "Queued", + "createdTime": "2026-11-20T10:00:00Z", "updatedTime": "2026-11-20T10:00:00Z", + "request": {"requestUrl": f"https://vsp.fake{req.path}", + "requestMethod": req.method, + "requestBody": req.body.decode() if req.body else ""}, + "affectedResources": [], + } + self._pending[jid] = execute + self._polls_left[jid] = self.job_polls + return Reply(202, dict(self.jobs[jid])) + + @route("GET", _P + r"/jobs/(?P\d+)") + def get_job(self, req, sdid, jid): + self._auth(req, sdid) + job = self.jobs.get(int(jid)) + if job is None: + raise _err(404, f"Job {jid} does not exist.", "KART40050-E") + jid_i = int(jid) + if jid_i in self._pending: + if self._polls_left[jid_i] > 0: + self._polls_left[jid_i] -= 1 + job.update(status="Running", state="Started") + else: + execute = self._pending.pop(jid_i) + try: + job.update(status="Completed", state="Succeeded", + affectedResources=execute(), + completedTime="2026-11-20T10:00:05Z") + except FakeError as e: + job.update(status="Completed", state="Failed", + error=e.reply.body, completedTime="2026-11-20T10:00:05Z") + return Reply(200, dict(job)) + + # -- resource-group lock ------------------------------------------------------- + + @route("POST", _P + r"/services/resource-group-service/actions/(?Plock|unlock)/invoke") + def rg_lock(self, req, sdid, act): + token = self._auth(req, sdid) + prefix = self._prefix(sdid) + + def run() -> list[str]: + if act == "lock": + if self.lock_owner not in (None, token): + raise _storage_err("The resource group is locked by another user.", + "2E10", "A002") + self.lock_owner = token + elif self.lock_owner == token: + self.lock_owner = None + return [f"{prefix}/services/resource-group-service"] + + return self._job(req, sdid, token, run, lock_request=True) + + # -- LDEVs ------------------------------------------------------------------ + + def _ldev_view(self, lid: int) -> dict[str, Any]: + ldev = self.ldevs.get(lid) + if ldev is None: + return {"ldevId": lid, "emulationType": NOT_DEFINED, "ssid": "0004", + "resourceGroupId": 0} + paths = [{"portId": port, "hostGroupNumber": num, "hostGroupName": g["name"], + "lun": lun} + for (port, num), g in sorted(self.groups.items()) + for lun, target in sorted(g["luns"].items()) if target == lid] + view = { + "ldevId": lid, "clprId": 0, "emulationType": "OPEN-V-CVS", + "byteFormatCapacity": f"{ldev['blocks'] * 512 / (1 << 30):.2f} G", + "blockCapacity": ldev["blocks"], "numOfPorts": len(paths), + "attributes": ["CVS", "HDP"], "label": ldev["label"], "status": "NML", + "mpBladeId": 0, "ssid": "0004", "poolId": ldev["pool"], + "numOfUsedBlock": 0, "isFullAllocationEnabled": False, + "resourceGroupId": 0, "dataReductionStatus": "DISABLED", + "dataReductionMode": "disabled", "isAluaEnabled": False, + "naaId": self.naa(lid), + } + if paths: + view["ports"] = paths + return view + + def _ldev_id(self, raw: str) -> int: + lid = int(raw) + if lid > _MAX_LDEV: + raise _err(400, f"The value of ldevId ({lid}) is out of range.", "KART40042-E") + return lid + + @route("GET", _P + r"/ldevs/(?P\d+)") + def get_ldev(self, req, sdid, lid): + self._auth(req, sdid) + return Reply(200, self._ldev_view(self._ldev_id(lid))) + + @route("GET", _P + r"/ldevs") + def list_ldevs(self, req, sdid): + self._auth(req, sdid) + option = req.q("ldevOption") + pool = req.q("poolId") + if pool is not None and option != "dpVolume": + raise _err(400, "poolId can be specified only with ldevOption=dpVolume.", + "KART40044-E") + if option not in (None, "defined", "dpVolume"): + raise _err(400, f"fake: ldevOption={option} not implemented", "KART40044-E") + count = int(req.q("count", "100") or 100) + if not 1 <= count <= self.max_count: + raise _err(400, f"The value of count ({count}) is out of range.", "KART40042-E") + head = int(req.q("headLdevId", "0") or 0) + data = [self._ldev_view(lid) for lid in sorted(self.ldevs) + if lid >= head and (pool is None or self.ldevs[lid]["pool"] == int(pool))] + return Reply(200, {"data": data[:count]}) + + @staticmethod + def _blocks(body: dict[str, Any]) -> int: + if "blockCapacity" in body: + return int(body["blockCapacity"]) + m = re.match(r"^(\d+)([KMGT])$", str(body.get("byteFormatCapacity", ""))) + if not m: + raise _err(400, "Specify blockCapacity or byteFormatCapacity.", "KART40041-E") + return int(m.group(1)) * _UNITS[m.group(2)] // 512 + + @route("POST", _P + r"/ldevs") + def post_ldev(self, req, sdid): + token = self._auth(req, sdid) + body = req.json() or {} + prefix = self._prefix(sdid) + if int(body.get("poolId", -1)) not in self.pools: + raise _err(400, f"The specified pool ({body.get('poolId')}) does not exist.", + "KART40045-E") + blocks = self._blocks(body) + if blocks <= 0: + raise _err(400, "The capacity is invalid.", "KART40042-E") + if ("startLdevId" in body) != ("endLdevId" in body): + raise _err(400, "Specify both startLdevId and endLdevId.", "KART40041-E") + + def run() -> list[str]: + if "ldevId" in body: + lid = int(body["ldevId"]) + if lid in self.ldevs: + raise _storage_err(f"LDEV {lid} is already defined.", "2E11", "0011") + else: + lid = self._free_ldev(int(body.get("startLdevId", 0)), + int(body.get("endLdevId", _MAX_LDEV))) + self.ldevs[lid] = {"pool": int(body["poolId"]), "blocks": blocks, "label": ""} + return [f"{prefix}/ldevs/{lid}"] + + return self._job(req, sdid, token, run) + + @route("PUT", _P + r"/ldevs/(?P\d+)") + def put_ldev(self, req, sdid, lid): + token = self._auth(req, sdid) + body = req.json() or {} + lid_i = self._ldev_id(lid) + prefix = self._prefix(sdid) + label = body.get("label") + if label is None: + raise _err(400, "fake: only label changes are implemented", "KART40041-E") + if len(label) > 32: + raise _err(400, "The label must be 32 characters or fewer.", "KART40042-E") + + def run() -> list[str]: + if lid_i not in self.ldevs: + raise _storage_err(f"LDEV {lid_i} is not defined.", "2E10", "0000") + self.ldevs[lid_i]["label"] = label + return [f"{prefix}/ldevs/{lid_i}"] + + return self._job(req, sdid, token, run) + + @route("POST", _P + r"/ldevs/(?P\d+)/actions/expand/invoke") + def expand_ldev(self, req, sdid, lid): + token = self._auth(req, sdid) + lid_i = self._ldev_id(lid) + params = (req.json() or {}).get("parameters") or {} + prefix = self._prefix(sdid) + add = int(params.get("additionalBlockCapacity", 0)) + if add <= 0: + raise _err(400, "additionalBlockCapacity must be positive.", "KART40042-E") + + def run() -> list[str]: + if lid_i not in self.ldevs: + raise _storage_err(f"LDEV {lid_i} is not defined.", "2E10", "0000") + self.ldevs[lid_i]["blocks"] += add + return [f"{prefix}/ldevs/{lid_i}"] + + return self._job(req, sdid, token, run) + + def _paths_of(self, lid: int) -> int: + return sum(1 for g in self.groups.values() for t in g["luns"].values() if t == lid) + + @route("DELETE", _P + r"/ldevs/(?P\d+)") + def delete_ldev(self, req, sdid, lid): + token = self._auth(req, sdid) + lid_i = self._ldev_id(lid) + prefix = self._prefix(sdid) + + def run() -> list[str]: + if lid_i not in self.ldevs: + raise _storage_err(f"LDEV {lid_i} is not defined.", "2E10", "0000") + if self._paths_of(lid_i): + raise _storage_err(f"LU paths are defined for LDEV {lid_i}.", "2E11", "0007") + del self.ldevs[lid_i] + return [f"{prefix}/ldevs/{lid_i}"] + + return self._job(req, sdid, token, run) + + # -- ports and host groups ------------------------------------------------------- + + def _port(self, port: str | None) -> str: + if port not in self.ports: + raise _err(404, f"The specified port ({port}) does not exist.", "KART40050-E") + return port # type: ignore[return-value] + + @route("GET", _P + r"/ports/(?P[^/]+)") + def get_port(self, req, sdid, port): + self._auth(req, sdid) + self._port(port) + info: dict[str, Any] = {"portId": port, "portType": self.ports[port], + "portAttributes": ["TAR"], "portSpeed": "AUT", + "lunSecuritySetting": True} + if self.ports[port] == "FIBRE": + info.update(loopId="EF", fabricMode=True, portConnection="PtoP", + wwn=f"50060e80{self.serial:06x}{list(self.ports).index(port):02x}") + else: + info.update(ipv4Address="192.0.2.20", tcpPort="3260") + return Reply(200, info) + + def _group_view(self, port: str, num: int) -> dict[str, Any]: + g = self.groups[(port, num)] + view: dict[str, Any] = {"hostGroupId": f"{port},{num}", "portId": port, + "hostGroupNumber": num, "hostGroupName": g["name"], + "hostMode": g["mode"], "resourceGroupId": 0} + if g["modeOptions"]: + view["hostModeOptions"] = list(g["modeOptions"]) + if self.ports[port] == "ISCSI": + view["iscsiName"] = f"iqn.1994-04.jp.co.hitachi:rsd.h8m.t.{self.serial:05d}." \ + f"{port[2:].replace('-', '')}{num:03x}" + return view + + def _existing_group(self, port: str | None, num: Any) -> tuple[str, int]: + port = self._port(port) + try: + key = (port, int(num)) + except (TypeError, ValueError) as e: + raise _err(400, "hostGroupNumber is required.", "KART40041-E") from e + if key not in self.groups: + raise _err(404, f"Host group {port},{num} does not exist.", "KART40050-E") + return key + + @route("GET", _P + r"/host-groups") + def list_groups(self, req, sdid): + self._auth(req, sdid) + port = self._port(req.q("portId")) + return Reply(200, {"data": [self._group_view(p, n) for (p, n) in sorted(self.groups) + if p == port]}) + + @route("POST", _P + r"/host-groups") + def post_group(self, req, sdid): + token = self._auth(req, sdid) + body = req.json() or {} + port = self._port(body.get("portId")) + name = str(body.get("hostGroupName") or "") + prefix = self._prefix(sdid) + limit = 32 if self.ports[port] == "ISCSI" else 64 + if not name or len(name) > limit: + raise _err(400, f"hostGroupName must be 1-{limit} characters.", "KART40042-E") + mode = body.get("hostMode", "LINUX/IRIX") + if mode not in ("LINUX/IRIX", "VMWARE_EX", "WIN_EX", "AIX", "HP-UX", "SOLARIS"): + raise _err(400, f"The hostMode ({mode}) is invalid.", "KART40042-E") + + def run() -> list[str]: + if any(p == port and g["name"] == name for (p, _), g in self.groups.items()): + raise _storage_err(f"The host group name {name} is already used on {port}.", + "2E11", "4E00") + num = body.get("hostGroupNumber") + if num is None: + num = next(n for n in range(1, 255) if (port, n) not in self.groups) + self.groups[(port, int(num))] = self._group(name, mode) + self.groups[(port, int(num))]["modeOptions"] = list(body.get("hostModeOptions", [])) + return [f"{prefix}/host-groups/{port},{num}"] + + return self._job(req, sdid, token, run) + + @route("DELETE", _P + r"/host-groups/(?P[^/,]+),(?P\d+)") + def delete_group(self, req, sdid, port, num): + token = self._auth(req, sdid) + key = self._existing_group(port, num) + prefix = self._prefix(sdid) + if key[1] == 0: + raise _err(400, "Host group 0 cannot be deleted.", "KART40042-E") + + def run() -> list[str]: + if self.groups[key]["luns"]: + raise _storage_err(f"LU paths are defined in host group {port},{num}.", + "2E11", "0007") + del self.groups[key] + return [f"{prefix}/host-groups/{port},{num}"] + + return self._job(req, sdid, token, run) + + # -- initiators -------------------------------------------------------------------- + + @route("GET", _P + r"/host-wwns") + def list_wwns(self, req, sdid): + self._auth(req, sdid) + port, num = self._existing_group(req.q("portId"), req.q("hostGroupNumber")) + g = self.groups[(port, num)] + return Reply(200, {"data": [ + {"hostWwnId": f"{port},{num},{w}", "portId": port, "hostGroupNumber": num, + "hostGroupName": g["name"], "hostWwn": w, "wwnNickname": "-"} + for w in g["wwns"]]}) + + @route("POST", _P + r"/host-wwns") + def post_wwn(self, req, sdid): + token = self._auth(req, sdid) + body = req.json() or {} + port, num = self._existing_group(body.get("portId"), body.get("hostGroupNumber")) + wwn = str(body.get("hostWwn", "")).lower() + prefix = self._prefix(sdid) + if self.ports[port] != "FIBRE": + raise _err(400, f"Port {port} is not a Fibre Channel port.", "KART40042-E") + if not re.match(r"^[0-9a-f]{16}$", wwn): + raise _err(400, f"The hostWwn ({wwn}) is invalid.", "KART40042-E") + + def run() -> list[str]: + for (p, n), g in self.groups.items(): + if p == port and wwn in g["wwns"]: + raise _storage_err( + f"The WWN {wwn} is already registered in host group {p},{n}.", + "B957", "4184") + self.groups[(port, num)]["wwns"].append(wwn) + return [f"{prefix}/host-wwns/{port},{num},{wwn}"] + + return self._job(req, sdid, token, run) + + @route("GET", _P + r"/host-iscsis") + def list_iscsis(self, req, sdid): + self._auth(req, sdid) + port, num = self._existing_group(req.q("portId"), req.q("hostGroupNumber")) + g = self.groups[(port, num)] + return Reply(200, {"data": [ + {"hostIscsiId": f"{port},{num},{n}", "portId": port, "hostGroupNumber": num, + "hostGroupName": g["name"], "iscsiName": n, "iscsiNickname": "-"} + for n in g["iscsis"]]}) + + @route("POST", _P + r"/host-iscsis") + def post_iscsi(self, req, sdid): + token = self._auth(req, sdid) + body = req.json() or {} + port, num = self._existing_group(body.get("portId"), body.get("hostGroupNumber")) + name = str(body.get("iscsiName", "")) + prefix = self._prefix(sdid) + if self.ports[port] != "ISCSI": + raise _err(400, f"Port {port} is not an iSCSI port.", "KART40042-E") + if not name.startswith(("iqn.", "eui.")): + raise _err(400, f"The iscsiName ({name}) is invalid.", "KART40042-E") + + def run() -> list[str]: + g = self.groups[(port, num)] + # The same initiator may log in to several targets of a port; only + # a duplicate within one target is refused. + if name.lower() in (n.lower() for n in g["iscsis"]): + raise _storage_err(f"The iSCSI name {name} is already registered.", + "B957", "4185") + g["iscsis"].append(name) + return [f"{prefix}/host-iscsis/{port},{num},{name}"] + + return self._job(req, sdid, token, run) + + # -- LU paths --------------------------------------------------------------------- + + @route("GET", _P + r"/luns") + def list_luns(self, req, sdid): + self._auth(req, sdid) + port, num = self._existing_group(req.q("portId"), req.q("hostGroupNumber")) + g = self.groups[(port, num)] + return Reply(200, {"data": [ + {"lunId": f"{port},{num},{lun}", "portId": port, "hostGroupNumber": num, + "hostMode": g["mode"], "lun": lun, "ldevId": lid, "isCommandDevice": False, + "luHostReserve": {"openSystem": False, "persistent": False, + "pgrKey": False, "mainframe": False, "acaReserve": False}} + for lun, lid in sorted(g["luns"].items())]}) + + @route("POST", _P + r"/luns") + def post_lun(self, req, sdid): + token = self._auth(req, sdid) + body = req.json() or {} + port, num = self._existing_group(body.get("portId"), body.get("hostGroupNumber")) + lid = int(body.get("ldevId", -1)) + prefix = self._prefix(sdid) + lun = body.get("lun") + if lun is not None and not 0 <= int(lun) <= 2047: + raise _err(400, f"The value of lun ({lun}) is out of range.", "KART40042-E") + + def run() -> list[str]: + g = self.groups[(port, num)] + if lid not in self.ldevs: + raise _storage_err(f"LDEV {lid} is not defined.", "2E10", "0000") + if lid in g["luns"].values(): + raise _storage_err(f"LDEV {lid} is already mapped in host group " + f"{port},{num}.", "B958", "0233") + n = int(lun) if lun is not None else next( + i for i in range(2048) if i not in g["luns"]) + if n in g["luns"]: + raise _storage_err(f"LUN {n} is already in use in host group " + f"{port},{num}.", "B958", "0234") + g["luns"][n] = lid + return [f"{prefix}/luns/{port},{num},{n}"] + + return self._job(req, sdid, token, run) + + @route("DELETE", _P + r"/luns/(?P[^/,]+),(?P\d+),(?P\d+)") + def delete_lun(self, req, sdid, port, num, lun): + token = self._auth(req, sdid) + key = self._existing_group(port, num) + prefix = self._prefix(sdid) + if int(lun) not in self.groups[key]["luns"]: + raise _err(404, f"LU path {port},{num},{lun} does not exist.", "KART40050-E") + + def run() -> list[str]: + self.groups[key]["luns"].pop(int(lun), None) + return [f"{prefix}/luns/{port},{num},{lun}"] + + return self._job(req, sdid, token, run) + + +def contract_setup(**options: Any) -> ContractSetup: + fake = FakeHitachiVsp(username="sbadmin", password="vsp-secret") + opts: dict[str, Any] = {"pool_id": 0, "ports": ["CL1-A", "CL2-A", "CL1-B", "CL2-B"]} + opts.update(options) + return ContractSetup( + fake=fake, + config=make_config("hitachi-vsp", "https://vsp.fake.example", **opts), + credential=Credential({"username": SecretStr("sbadmin"), + "password": SecretStr("vsp-secret")}), + bad_credential=Credential({"username": SecretStr("sbadmin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_ldev(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/hpe_nimble.py b/simplyblock_core/storage_arrays/testing/hpe_nimble.py new file mode 100644 index 0000000000..f02e5a9728 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/hpe_nimble.py @@ -0,0 +1,363 @@ +"""Fake HPE Alletra 5000/6000 / Nimble, NimbleOS REST v1 (subset the adapter uses). + +Quirks reproduced: tokens from ``POST /v1/tokens`` checked on ``X-Auth-Token`` +and released by token *id*; ``{"data": ...}`` envelopes on requests and +responses; ``{"messages": [{"code": "SM_...", ...}]}`` errors; 42-hex object +ids; sizes in MiB; volumes that must be offline before deletion and cannot be +offlined or deleted while ACRs exist; FC WWPNs in colon form; and LUN 0 for +every iSCSI ACR with volume-scoped targets. +""" +from __future__ import annotations + +import re +import secrets +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_ID = r"(?P(?!detail$)[^/]+)" +_ID_RE = re.compile(r"^[0-9a-f]{42}$") +_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.:\-]{0,214}$") +_WWPN_RE = re.compile(r"^([0-9a-f]{2}:){7}[0-9a-f]{2}$") +_NG_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.:\-]{0,63}$") + + +def _err(status: int, code: str, text: str) -> FakeError: + return FakeError(status, {"messages": [{"code": code, "severity": "error", "text": text}]}) + + +class FakeHpeNimble(FakeArray): + vendor = "hpe-nimble" + + def __init__(self, username: str = "admin", password: str = "admin", + volume_scoped_targets: bool = True): + super().__init__() + self.username = username + self.password = password + self.volume_scoped_targets = volume_scoped_targets + self.tokens: dict[str, str] = {} # token id -> session_token + self.pools = {self._new_id(0x0a): "default", self._new_id(0x0a): "fast"} + self.perfpolicies = {self._new_id(0x03): "default", self._new_id(0x03): "Oracle OLTP"} + self.volumes: dict[str, dict[str, Any]] = {} # id -> volume + self.groups: dict[str, dict[str, Any]] = {} # id -> initiator group + self.initiators: dict[str, dict[str, Any]] = {} + self.acrs: dict[str, dict[str, Any]] = {} + + def _new_id(self, kind: int) -> str: + # NimbleOS ids: 42 hex chars, group-unique prefix + object type + counter. + return f"06{secrets.token_hex(9)}{kind:02x}{self.next_id():020x}"[:42] + + # -- fake controls ------------------------------------------------------ + + def expire_sessions(self) -> None: + self.tokens.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields["password"] + + def add_volume(self, name: str, size: int, pool_id: str | None = None, + perfpolicy_id: str | None = None) -> str: + vid = self._new_id(0x2a) + pool_id = pool_id or next(iter(self.pools)) + perfpolicy_id = perfpolicy_id or next(iter(self.perfpolicies)) + self.volumes[vid] = { + "id": vid, "name": name, "size": -(-size // (1 << 20)), + "serial_number": secrets.token_hex(8) + "6c9ce900" + secrets.token_hex(4), + "target_name": f"iqn.2007-11.com.nimblestorage:{name}-v{secrets.token_hex(8)}", + "online": True, "vol_state": "online", "pool_id": pool_id, + "pool_name": self.pools[pool_id], "perfpolicy_id": perfpolicy_id, + "perfpolicy_name": self.perfpolicies[perfpolicy_id], "access_control_records": [], + } + return vid + + # -- envelope helpers ----------------------------------------------------- + + def _auth(self, req: Request) -> None: + if req.headers.get("X-Auth-Token") not in self.tokens.values(): + raise _err(401, "SM_http_unauthorized", "Unauthorized: session token invalid") + + @staticmethod + def _body(req: Request) -> dict[str, Any]: + body = req.json() + if not isinstance(body, dict) or not isinstance(body.get("data"), dict): + raise _err(400, "SM_missing_arg", "Missing data envelope") + return body["data"] + + @staticmethod + def _page(items: list[dict[str, Any]]) -> Reply: + return Reply(200, {"startRow": 0, "endRow": len(items), "totalRows": len(items), + "data": items}) + + @staticmethod + def _filter(req: Request, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + out = items + for key, values in req.query.items(): + if key in ("startRow", "endRow", "pageSize", "fields", "sortBy"): + continue + out = [i for i in out if str(i.get(key, "")).lower() == values[0].lower()] + return out + + def _get(self, table: dict[str, dict[str, Any]], obj_id: str, what: str) -> dict[str, Any]: + if not _ID_RE.match(obj_id): + raise _err(400, "SM_invalid_arg", f"Invalid {what} id: {obj_id}") + if obj_id not in table: + raise _err(404, "SM_http_not_found", f"Object not found: {what} {obj_id}") + return table[obj_id] + + # -- auth / info ---------------------------------------------------------- + + @route("POST", "/v1/tokens") + def login(self, req): + data = self._body(req) + if data.get("username") != self.username or data.get("password") != self.password: + raise _err(401, "SM_http_unauthorized", "Unauthorized: invalid credentials") + tid, token = self._new_id(0x1f), secrets.token_hex(16) + self.tokens[tid] = token + return Reply(201, {"data": {"id": tid, "session_token": token, + "username": self.username, "app_name": "", + "source_ip": "192.0.2.1"}}) + + @route("DELETE", "/v1/tokens/" + _ID) + def logout(self, req, obj_id): + self._auth(req) + self._get(self.tokens, obj_id, "token") # type: ignore[arg-type] + del self.tokens[obj_id] + return Reply(200, {}) + + @route("GET", "/v1/arrays/detail") + def arrays(self, req): + self._auth(req) + return self._page([{"id": "0906b8bc6e4a3c2a7f000000000000000000000001", + "name": "fake-af40", "model": "AF40", "serial": "AF-123456", + "version": "6.1.2.100-1010", "status": "reachable"}]) + + @route("GET", "/v1/pools/detail") + def pools_detail(self, req): + self._auth(req) + return self._page(self._filter(req, [{"id": k, "name": v} + for k, v in self.pools.items()])) + + @route("GET", "/v1/performance_policies/detail") + def perfpolicies_detail(self, req): + self._auth(req) + return self._page(self._filter(req, [{"id": k, "name": v} + for k, v in self.perfpolicies.items()])) + + # -- volumes ------------------------------------------------------------- + + @route("POST", "/v1/volumes") + def post_volumes(self, req): + self._auth(req) + data = self._body(req) + name, size = data.get("name"), data.get("size") + if not isinstance(name, str) or not _NAME_RE.match(name): + raise _err(400, "SM_invalid_arg", "Invalid argument: name") + if not isinstance(size, int) or size < 1: + raise _err(400, "SM_invalid_arg", "Invalid argument: size (MiB)") + if any(v["name"] == name for v in self.volumes.values()): + raise _err(409, "SM_eexist", f"Object exists: volume {name}") + pool_id, pp_id = data.get("pool_id"), data.get("perfpolicy_id") + if pool_id is not None and pool_id not in self.pools: + raise _err(404, "SM_http_not_found", f"Object not found: pool {pool_id}") + if pp_id is not None and pp_id not in self.perfpolicies: + raise _err(404, "SM_http_not_found", f"Object not found: perfpolicy {pp_id}") + vid = self.add_volume(name, size << 20, pool_id, pp_id) + return Reply(201, {"data": self.volumes[vid]}) + + @route("GET", "/v1/volumes/detail") + def volumes_detail(self, req): + self._auth(req) + return self._page(self._filter(req, list(self.volumes.values()))) + + @route("GET", "/v1/volumes/" + _ID) + def get_volume(self, req, obj_id): + self._auth(req) + return Reply(200, {"data": self._get(self.volumes, obj_id, "volume")}) + + @route("PUT", "/v1/volumes/" + _ID) + def put_volume(self, req, obj_id): + self._auth(req) + vol = self._get(self.volumes, obj_id, "volume") + data = self._body(req) + if "size" in data: + size = data["size"] + if not isinstance(size, int) or size < vol["size"]: + raise _err(400, "SM_vol_size_decreased", + "Volume size cannot be decreased") + vol["size"] = size + if "online" in data: + if not data["online"] and self._vol_acrs(obj_id) and not data.get("force"): + raise _err(409, "SM_vol_has_connections", + "Volume has access control records; cannot offline") + vol["online"] = bool(data["online"]) + vol["vol_state"] = "online" if vol["online"] else "offline" + return Reply(200, {"data": vol}) + + @route("DELETE", "/v1/volumes/" + _ID) + def delete_volume(self, req, obj_id): + self._auth(req) + vol = self._get(self.volumes, obj_id, "volume") + if vol["online"]: + raise _err(409, "SM_vol_online", "Volume must be offline before it can be deleted") + if self._vol_acrs(obj_id): + raise _err(409, "SM_vol_has_acl", "Volume has access control records") + del self.volumes[obj_id] + return Reply(200, {}) + + # -- initiator groups / initiators ---------------------------------------------- + + def _group_view(self, g: dict[str, Any]) -> dict[str, Any]: + members = [i for i in self.initiators.values() if i["initiator_group_id"] == g["id"]] + view = dict(g) + if g["access_protocol"] == "iscsi": + view["iscsi_initiators"] = [{"id": i["id"], "label": i["label"], "iqn": i["iqn"], + "ip_address": "*"} for i in members] + view["fc_initiators"] = None + else: + view["fc_initiators"] = [{"wwpn": i["wwpn"], "alias": ""} for i in members] + view["iscsi_initiators"] = None + return view + + @route("GET", "/v1/initiator_groups/detail") + def groups_detail(self, req): + self._auth(req) + return self._page(self._filter(req, [self._group_view(g) + for g in self.groups.values()])) + + @route("GET", "/v1/initiator_groups/" + _ID) + def get_group(self, req, obj_id): + self._auth(req) + return Reply(200, {"data": self._group_view(self._get(self.groups, obj_id, "group"))}) + + @route("POST", "/v1/initiator_groups") + def post_group(self, req): + self._auth(req) + data = self._body(req) + name, proto = data.get("name"), data.get("access_protocol") + if not isinstance(name, str) or not _NG_NAME_RE.match(name): + raise _err(400, "SM_invalid_arg", "Invalid argument: name") + if proto not in ("iscsi", "fc"): + raise _err(400, "SM_invalid_arg", "Invalid argument: access_protocol") + if any(g["name"] == name for g in self.groups.values()): + raise _err(409, "SM_eexist", f"Object exists: initiator group {name}") + gid = self._new_id(0x02) + self.groups[gid] = {"id": gid, "name": name, "access_protocol": proto, + "description": data.get("description", ""), + "target_subnets": [], "host_type": "auto"} + return Reply(201, {"data": self._group_view(self.groups[gid])}) + + @route("DELETE", "/v1/initiator_groups/" + _ID) + def delete_group(self, req, obj_id): + self._auth(req) + self._get(self.groups, obj_id, "group") + if any(a["initiator_group_id"] == obj_id for a in self.acrs.values()): + raise _err(409, "SM_igrp_has_acl", + "Initiator group is in use by access control records") + for iid in [i for i, v in self.initiators.items() if v["initiator_group_id"] == obj_id]: + del self.initiators[iid] + del self.groups[obj_id] + return Reply(200, {}) + + @route("GET", "/v1/initiators/detail") + def initiators_detail(self, req): + self._auth(req) + return self._page(self._filter(req, list(self.initiators.values()))) + + @route("POST", "/v1/initiators") + def post_initiator(self, req): + self._auth(req) + data = self._body(req) + gid, proto = data.get("initiator_group_id"), data.get("access_protocol") + group = self._get(self.groups, str(gid), "group") + if proto != group["access_protocol"]: + raise _err(400, "SM_invalid_arg", "access_protocol does not match initiator group") + item = {"id": self._new_id(0x04), "access_protocol": proto, + "initiator_group_id": gid, "initiator_group_name": group["name"], + "label": "", "iqn": "", "ip_address": "", "wwpn": "", "alias": ""} + if proto == "iscsi": + iqn, label = data.get("iqn"), data.get("label") + if not iqn or not label or len(label) > 64: + raise _err(400, "SM_missing_arg", "iqn and label are required") + if any(i["iqn"].lower() == iqn.lower() for i in self.initiators.values()): + raise _err(409, "SM_eexist", f"Object exists: initiator {iqn}") + item.update(iqn=iqn, label=label, ip_address=data.get("ip_address", "*")) + else: + wwpn = str(data.get("wwpn", "")) + if not _WWPN_RE.match(wwpn): + raise _err(400, "SM_invalid_arg", "Invalid argument: wwpn (colon form)") + if any(i["wwpn"] == wwpn for i in self.initiators.values()): + raise _err(409, "SM_eexist", f"Object exists: initiator {wwpn}") + item.update(wwpn=wwpn, alias=data.get("alias", "")) + self.initiators[item["id"]] = item + return Reply(201, {"data": item}) + + # -- access control records ------------------------------------------------ + + def _vol_acrs(self, vol_id: str) -> list[dict[str, Any]]: + return [a for a in self.acrs.values() if a["vol_id"] == vol_id] + + @route("GET", "/v1/access_control_records/detail") + def acrs_detail(self, req): + self._auth(req) + return self._page(self._filter(req, list(self.acrs.values()))) + + @route("POST", "/v1/access_control_records") + def post_acr(self, req): + self._auth(req) + data = self._body(req) + vol = self._get(self.volumes, str(data.get("vol_id")), "volume") + group = self._get(self.groups, str(data.get("initiator_group_id")), "group") + if data.get("apply_to", "both") not in ("volume", "snapshot", "both", "pe", "vvol_volume", + "vvol_snapshot"): + raise _err(400, "SM_invalid_arg", "Invalid argument: apply_to") + if any(a["initiator_group_id"] == group["id"] for a in self._vol_acrs(vol["id"])): + raise _err(409, "SM_eexist", "Object exists: access control record") + if group["access_protocol"] == "iscsi" and self.volume_scoped_targets: + lun = 0 + else: + used = {a["lun"] for a in self.acrs.values() + if a["initiator_group_id"] == group["id"]} + lun = next(n for n in range(2048) if n not in used) + aid = self._new_id(0x0d) + self.acrs[aid] = {"id": aid, "vol_id": vol["id"], "vol_name": vol["name"], + "initiator_group_id": group["id"], + "initiator_group_name": group["name"], + "apply_to": data.get("apply_to", "both"), "lun": lun, + "chap_user_id": "", "pe_id": "", "snap_id": ""} + return Reply(201, {"data": self.acrs[aid]}) + + @route("DELETE", "/v1/access_control_records/" + _ID) + def delete_acr(self, req, obj_id): + self._auth(req) + self._get(self.acrs, obj_id, "access control record") + del self.acrs[obj_id] + return Reply(200, {}) + + +def contract_setup() -> ContractSetup: + fake = FakeHpeNimble(username="admin", password="nimble-pw") + return ContractSetup( + fake=fake, + config=make_config("hpe-nimble", "https://nimble.fake.example:5392"), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("nimble-pw")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/hpe_wsapi.py b/simplyblock_core/storage_arrays/testing/hpe_wsapi.py new file mode 100644 index 0000000000..9b93d51bf3 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/hpe_wsapi.py @@ -0,0 +1,321 @@ +"""Fake HPE WSAPI (Alletra MP / Primera / 3PAR), the subset the adapter uses. + +Quirks reproduced: session keys created with ``POST /api/v1/credentials`` and +checked on ``X-HP3PAR-WSAPI-SessionKey``; ``{"code", "desc"}`` error bodies; +volumes addressed by name with sizes in MiB rounded up to 256 MiB; GROW +taking an increment; host requests using ``iSCSINames``/``FCWWNs`` while +responses report ``iSCSIPaths``/``FCPaths``; VLUN creation answering 201 +with an empty body and a ``Location`` header; and VLUN listings that contain +the template plus one active VLUN per host path. +""" +from __future__ import annotations + +import re +import secrets +from typing import Any +from urllib.parse import unquote + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_A = "/api/v1" +_UNIT_MIB = 256 +_MAX_NAME = 31 +_QUERY_RE = re.compile(r'^"?\s*volumeName\s+EQ\s+(?P\S+?)\s*"?$') + + +def _err(status: int, code: int, desc: str) -> FakeError: + return FakeError(status, {"code": code, "desc": desc}) + + +class FakeHpeWsapi(FakeArray): + vendor = "hpe-wsapi" + + def __init__(self, username: str = "3paradm", password: str = "3pardata", + cpgs: tuple[str, ...] = ("SSD_r6",), paths_per_host: int = 2): + super().__init__() + self.username = username + self.password = password + self.cpgs = set(cpgs) + #: active VLUN entries listed per mapping (one per host port path) + self.paths_per_host = paths_per_host + self.sessions: set[str] = set() + self.volumes: dict[str, dict[str, Any]] = {} # name -> volume + self.hosts: dict[str, dict[str, Any]] = {} # name -> host + self.vluns: list[dict[str, Any]] = [] # templates {volumeName, hostname, lun} + self.created_bodies: list[dict[str, Any]] = [] + + # -- fake controls ------------------------------------------------------ + + def expire_sessions(self) -> None: + self.sessions.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields["password"] + + def add_volume(self, name: str, size: int) -> str: + mib = -(-size // (1 << 20)) + mib = -(-mib // _UNIT_MIB) * _UNIT_MIB + self.volumes[name] = { + "id": self.next_id(), "name": name, "sizeMiB": mib, + "wwn": "60002AC" + secrets.token_hex(13).upper()[:25], + "provisioningType": 2, "userCPG": next(iter(self.cpgs)), "state": 1, + "uuid": secrets.token_hex(16), + } + return name + + # -- auth --------------------------------------------------------------- + + def _auth(self, req: Request) -> None: + if req.headers.get("X-HP3PAR-WSAPI-SessionKey") not in self.sessions: + raise _err(401, 6, "invalid session key") + + @route("POST", _A + "/credentials") + def login(self, req): + body = req.json() or {} + if body.get("user") != self.username or body.get("password") != self.password: + raise _err(403, 5, "invalid username or password") + key = "0-" + secrets.token_hex(16) + "-" + secrets.token_hex(4) + self.sessions.add(key) + return Reply(201, {"key": key}) + + @route("DELETE", _A + "/credentials/(?P[^/]+)") + def logout(self, req, key): + key = unquote(key) + if key not in self.sessions: + raise _err(404, 6, "invalid session key") + self.sessions.discard(key) + return Reply(200) + + @route("GET", _A + "/system") + def system(self, req): + self._auth(req) + return Reply(200, {"id": 412345, "name": "fake-b10000", "systemVersion": "10.4.0.123", + "model": "HPE Alletra Storage MP B10000", + "serialNumber": "CZ12345678", "IPv4Addr": "192.0.2.10"}) + + # -- volumes ------------------------------------------------------------- + + def _vol(self, name: str) -> dict[str, Any]: + name = unquote(name) + if name not in self.volumes: + raise _err(404, 23, "volume does not exist") + return self.volumes[name] + + @route("POST", _A + "/volumes") + def post_volumes(self, req): + self._auth(req) + body = req.json() or {} + self.created_bodies.append(body) + name, cpg, size = body.get("name"), body.get("cpg"), body.get("sizeMiB") + if not name or not cpg or not isinstance(size, int): + raise _err(400, 15, "missing required field") + if len(name) > _MAX_NAME: + raise _err(400, 21, "invalid input: string length exceeds limit") + if cpg not in self.cpgs: + raise _err(404, 15, "CPG does not exist") + if name in self.volumes: + raise _err(409, 22, "volume exists") + if size < 1: + raise _err(400, 40, "invalid input: number exceeds expected range") + self.add_volume(name, size << 20) + return Reply(201, None, {"Location": f"{_A}/volumes/{name}"}) + + @route("GET", _A + "/volumes/(?P[^/]+)") + def get_volume(self, req, name): + self._auth(req) + return Reply(200, self._vol(name)) + + @route("PUT", _A + "/volumes/(?P[^/]+)") + def put_volume(self, req, name): + self._auth(req) + vol = self._vol(name) + body = req.json() or {} + if body.get("action") == 3: + delta = body.get("sizeMiB") + if not isinstance(delta, int) or delta <= 0: + raise _err(400, 40, "invalid input: number exceeds expected range") + vol["sizeMiB"] = -(-(vol["sizeMiB"] + delta) // _UNIT_MIB) * _UNIT_MIB + return Reply(200) + raise _err(400, 29, "invalid input: unsupported action") + + @route("DELETE", _A + "/volumes/(?P[^/]+)") + def delete_volume(self, req, name): + self._auth(req) + vol = self._vol(name) + if any(v["volumeName"] == vol["name"] for v in self.vluns): + raise _err(409, 34, "volume is exported (has VLUNs)") + del self.volumes[vol["name"]] + return Reply(200) + + # -- hosts ---------------------------------------------------------------- + + def _host(self, name: str) -> dict[str, Any]: + name = unquote(name) + if name not in self.hosts: + raise _err(404, 17, "host does not exist") + return self.hosts[name] + + def _claim(self, host: str, body: dict[str, Any]) -> None: + iqns = [str(i) for i in body.get("iSCSINames") or []] + wwns = [str(w).upper().replace(":", "") for w in body.get("FCWWNs") or []] + for w in wwns: + if not re.fullmatch(r"[0-9A-F]{16}", w): + raise _err(400, 32, "invalid WWN") + for other in self.hosts.values(): + if other["name"] == host: + continue + taken = ({p["name"].lower() for p in other["iSCSIPaths"]} & {i.lower() for i in iqns} + or {p["wwn"] for p in other["FCPaths"]} & set(wwns)) + if taken: + raise _err(409, 73, f"path {min(taken)} is already used by " + f"host {other['name']}") + h = self.hosts[host] + for i in iqns: + if i.lower() not in {p["name"].lower() for p in h["iSCSIPaths"]}: + h["iSCSIPaths"].append({"name": i, "IPAddr": "0.0.0.0", "hostSpeed": 0}) + for w in wwns: + if w not in {p["wwn"] for p in h["FCPaths"]}: + h["FCPaths"].append({"wwn": w, "hostSpeed": 0}) + + @route("POST", _A + "/hosts") + def post_hosts(self, req): + self._auth(req) + body = req.json() or {} + name = body.get("name") + if not name: + raise _err(400, 15, "missing required field") + if len(name) > _MAX_NAME: + raise _err(400, 21, "invalid input: string length exceeds limit") + if name in self.hosts: + raise _err(409, 16, "host exists") + if body.get("iSCSINames") and body.get("FCWWNs"): + # Mirrors the adapter's conservative reading: one path type per request. + raise _err(400, 28, "invalid input: iSCSINames and FCWWNs are exclusive") + self.hosts[name] = {"id": self.next_id(), "name": name, + "persona": body.get("persona", 2), + "iSCSIPaths": [], "FCPaths": [], "descriptors": {}} + try: + self._claim(name, body) + except FakeError: + del self.hosts[name] + raise + return Reply(201, None, {"Location": f"{_A}/hosts/{name}"}) + + @route("GET", _A + "/hosts/(?P[^/]+)") + def get_host(self, req, name): + self._auth(req) + return Reply(200, self._host(name)) + + @route("PUT", _A + "/hosts/(?P[^/]+)") + def put_host(self, req, name): + self._auth(req) + host = self._host(name) + body = req.json() or {} + if body.get("pathOperation") != 1: + raise _err(400, 29, "invalid input: unsupported pathOperation") + if body.get("iSCSINames") and body.get("FCWWNs"): + raise _err(400, 28, "invalid input: iSCSINames and FCWWNs are exclusive") + self._claim(host["name"], body) + return Reply(200) + + @route("DELETE", _A + "/hosts/(?P[^/]+)") + def delete_host(self, req, name): + self._auth(req) + host = self._host(name) + if any(v["hostname"] == host["name"] for v in self.vluns): + raise _err(409, 26, "host has exported VLUNs") + del self.hosts[host["name"]] + return Reply(200) + + # -- VLUNs ------------------------------------------------------------------ + + def _vlun_members(self, vol: str | None) -> list[dict[str, Any]]: + out = [] + for t in self.vluns: + if vol is not None and t["volumeName"] != vol: + continue + v = self.volumes[t["volumeName"]] + base = {"lun": t["lun"], "volumeName": t["volumeName"], "hostname": t["hostname"], + "volumeWWN": v["wwn"], "type": 4} + out.append({**base, "active": False}) + for port in range(self.paths_per_host): + out.append({**base, "active": True, "remoteName": f"path{port}", + "portPos": {"node": port, "slot": 2, "cardPort": 1}}) + return out + + @route("GET", _A + "/vluns") + def get_vluns(self, req): + self._auth(req) + query = req.q("query") + vol = None + if query is not None: + m = _QUERY_RE.match(query) + if not m: + raise _err(400, 92, "invalid query") + vol = m["vol"] + members = self._vlun_members(vol) + return Reply(200, {"total": len(members), "members": members}) + + @route("POST", _A + "/vluns") + def post_vluns(self, req): + self._auth(req) + body = req.json() or {} + vol, host = body.get("volumeName"), body.get("hostname") + if vol not in self.volumes: + raise _err(404, 23, "volume does not exist") + if host not in self.hosts: + raise _err(404, 17, "host does not exist") + if any(t["volumeName"] == vol and t["hostname"] == host for t in self.vluns): + raise _err(409, 29, "VLUN exists") + used = {t["lun"] for t in self.vluns if t["hostname"] == host} + start = int(body.get("lun", 0)) + if body.get("autoLun"): + lun = next(n for n in range(start, 16384) if n not in used) + elif start in used: + raise _err(409, 29, "LUN already in use") + else: + lun = start + self.vluns.append({"volumeName": vol, "hostname": host, "lun": lun}) + return Reply(201, None, {"Location": f"{_A}/vluns/{vol},{lun},{host}"}) + + @route("DELETE", _A + "/vluns/(?P[^/]+)") + def delete_vlun(self, req, spec): + self._auth(req) + parts = unquote(spec).split(",") + if len(parts) != 3 or not parts[1].isdigit(): + raise _err(400, 28, "invalid VLUN id") + vol, lun, host = parts[0], int(parts[1]), parts[2] + for t in self.vluns: + if t["volumeName"] == vol and t["lun"] == lun and t["hostname"] == host: + self.vluns.remove(t) + return Reply(200) + raise _err(404, 19, "VLUN does not exist") + + +def contract_setup() -> ContractSetup: + fake = FakeHpeWsapi(username="3paradm", password="wsapi-pw") + return ContractSetup( + fake=fake, + config=make_config("hpe-wsapi", "https://wsapi.fake.example:443", cpg="SSD_r6"), + credential=Credential({"username": SecretStr("3paradm"), + "password": SecretStr("wsapi-pw")}), + bad_credential=Credential({"username": SecretStr("3paradm"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/huawei_dorado.py b/simplyblock_core/storage_arrays/testing/huawei_dorado.py new file mode 100644 index 0000000000..408f6d22f5 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/huawei_dorado.py @@ -0,0 +1,589 @@ +"""Fake Huawei OceanStor Dorado DeviceManager REST API (the subset the adapter uses). + +Quirks reproduced: + +* every answer is HTTP 200 with ``{"data": ..., "error": {"code", "description"}}``; + failures only show in ``error.code``; a query matching nothing omits ``data``; +* login at ``/deviceManager/rest/xx/sessions`` returns ``deviceid`` + + ``iBaseToken``; later calls must use ``/deviceManager/rest//`` and + the ``iBaseToken`` header, else ``-401``; wrong password is 1077987870; +* every value is a string; LUN ``CAPACITY`` is in 512-byte sectors; +* initiators are free-standing objects bound to a host via ``PARENTID``; +* mapping is host group + LUN group + mapping view; a host can be in one host + group only; host LUN ids are assigned when a LUN becomes visible to a host + and reported inside the ``ASSOCIATEMETADATA`` JSON string. + +Codes marked "unverified" below are stand-ins: the real array refuses those +operations, but we do not know with which code. The adapter must not depend +on them. +""" +from __future__ import annotations + +import json +import secrets +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_D = r"/deviceManager/rest/(?P[^/]+)" + +UNAUTHORIZED = -401 +WRONG_PASSWORD = 1077987870 +OBJECT_NOT_EXIST = 1077948996 +OBJECT_NAME_EXISTS = 1077948993 +OBJECT_ID_NOT_UNIQUE = 1077948997 +LUN_NOT_EXIST = 1077936859 +LUN_ALREADY_IN_GROUP = 1077936862 +HOST_NOT_EXIST = 1077937498 +HOSTGROUP_NOT_EXIST = 1077937500 +HOST_ALREADY_IN_GROUP = 1077937501 +HOST_NOT_IN_GROUP = 1073745412 +HOSTGROUP_NOT_IN_VIEW = 1073804552 +LUNGROUP_NOT_IN_VIEW = 1073804554 +HOSTGROUP_ALREADY_IN_VIEW = 1073804556 +LUNGROUP_ALREADY_IN_VIEW = 1073804560 +MAPPINGVIEW_NOT_EXIST = 1077951819 +PARAMETER_INCORRECT = 50331651 +#: unverified: object still in use (LUN in a group, host with initiators, ...) +IN_USE = 1077949000 +#: unverified: host already in another host group / initiator bound elsewhere +ALREADY_ASSOCIATED = 1077949002 + + +def _fail(code: int, description: str) -> FakeError: + return FakeError(200, {"data": {}, "error": {"code": code, "description": description, + "suggestion": ""}}) + + +def _ok(data: Any = None) -> Reply: + body: dict[str, Any] = {"error": {"code": 0, "description": "0"}} + if data is not None and data != []: + body["data"] = data + return Reply(200, body) + + +def _filter_name(req: Request) -> str | None: + f = req.q("filter") + if not f: + return None + key, _, value = f.partition("::") + if key != "NAME": + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + return value + + +class FakeHuaweiDorado(FakeArray): + vendor = "huawei-dorado" + + def __init__(self, username: str = "admin", password: str = "Admin@storage1", + pools: tuple[str, ...] = ("StoragePool001",)): + super().__init__() + self.username = username + self.password = password + self.device_id = "2102353GTH10L8000008" + self.tokens: set[str] = set() + self.pools = {str(i): {"ID": str(i), "NAME": n, "USAGETYPE": "1"} + for i, n in enumerate(pools)} + self.luns: dict[str, dict[str, Any]] = {} + self.hosts: dict[str, dict[str, Any]] = {} + self.hostgroups: dict[str, dict[str, Any]] = {} # + "hosts": set + self.lungroups: dict[str, dict[str, Any]] = {} # + "luns": set + self.views: dict[str, dict[str, Any]] = {} # + "hostgroups", "lungroups" + self.initiators: dict[str, dict[str, dict[str, Any]]] = { + "iscsi_initiator": {}, "fc_initiator": {}} + self.host_lun_ids: dict[tuple[str, str], int] = {} + + # -- fake controls ------------------------------------------------------------- + + def expire_sessions(self) -> None: + self.tokens.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields["username"] + self.password = fields["password"] + + def add_volume(self, name: str, size: int) -> str: + lid = str(self.next_id()) + pool = next(iter(self.pools.values())) + self.luns[lid] = { + "ID": lid, "NAME": name, "TYPE": "11", "PARENTID": pool["ID"], + "PARENTNAME": pool["NAME"], "CAPACITY": str(size // 512), "ALLOCTYPE": "1", + "SECTORSIZE": "512", "HEALTHSTATUS": "1", "RUNNINGSTATUS": "27", + "WWN": "6" + secrets.token_hex(4) + "1001f4e7d" + secrets.token_hex(5) + + f"{int(lid):08x}"[:4], + "DESCRIPTION": "", "ISADD2LUNGROUP": "false"} + return lid + + # -- auth ---------------------------------------------------------------------- + + def _auth(self, req: Request, dev: str) -> None: + if dev != self.device_id or req.headers.get("iBaseToken") not in self.tokens: + raise _fail(UNAUTHORIZED, "The user is not logged in or the session has expired.") + + @route("POST", r"/deviceManager/rest/xx/sessions") + def login(self, req): + body = req.json() or {} + if body.get("username") != self.username or body.get("password") != self.password: + raise _fail(WRONG_PASSWORD, "The user name or password is incorrect.") + if str(body.get("scope")) not in ("0", "1"): + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + token = secrets.token_hex(16) + self.tokens.add(token) + reply = _ok({"deviceid": self.device_id, "iBaseToken": token, "accountstate": 1, + "lastloginip": "10.0.0.1", "level": 2, "pwdchangetime": -1, + "usergroup": "", "userid": body["username"], "username": body["username"], + "userscope": "0"}) + reply.headers["Set-Cookie"] = f"session={secrets.token_hex(8)}; Secure; HttpOnly" + return reply + + @route("DELETE", _D + "/sessions") + def logout(self, req, dev): + self.tokens.discard(req.headers.get("iBaseToken")) + return _ok() + + @route("GET", _D + "/system/") + def system(self, req, dev): + self._auth(req, dev) + return _ok({"ID": self.device_id, "NAME": "fake-dorado", "TYPE": "201", + "PRODUCTMODE": "811", "productModeString": "OceanStor Dorado 5000 V6", + "PRODUCTVERSION": "V600R005C20", "pointRelease": "6.1.5", + "HEALTHSTATUS": "1", "RUNNINGSTATUS": "1"}) + + @route("GET", _D + "/storagepool") + def storagepool(self, req, dev): + self._auth(req, dev) + name = _filter_name(req) + return _ok([p for p in self.pools.values() if name is None or p["NAME"] == name]) + + # -- LUNs -------------------------------------------------------------------------- + + def _lun(self, lid: str) -> dict[str, Any]: + if lid not in self.luns: + raise _fail(LUN_NOT_EXIST, "The LUN does not exist.") + return self.luns[lid] + + @route("GET", _D + "/lun") + def get_luns(self, req, dev): + self._auth(req, dev) + name = _filter_name(req) + return _ok([lun for lun in self.luns.values() if name is None or lun["NAME"] == name]) + + @route("POST", _D + "/lun") + def post_lun(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + name, cap = b.get("NAME"), b.get("CAPACITY") + if not name or len(name) > 31 or not isinstance(cap, int) or cap <= 0: + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if str(b.get("PARENTID")) not in self.pools: + raise _fail(OBJECT_NOT_EXIST, "The specified storage pool does not exist.") + if any(lun["NAME"] == name for lun in self.luns.values()): + raise _fail(OBJECT_NAME_EXISTS, "The name already exists.") + lid = self.add_volume(name, cap * 512) + self.luns[lid]["PARENTID"] = str(b["PARENTID"]) + self.luns[lid]["PARENTNAME"] = self.pools[str(b["PARENTID"])]["NAME"] + self.luns[lid]["ALLOCTYPE"] = str(b.get("ALLOCTYPE", 1)) + return _ok(self.luns[lid]) + + @route("GET", _D + r"/lun/(?P\d+)") + def get_lun(self, req, dev, lid): + self._auth(req, dev) + return _ok(self._lun(lid)) + + @route("PUT", _D + "/lun/expand") + def expand_lun(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + lun = self._lun(str(b.get("ID"))) + cap = b.get("CAPACITY") + if not isinstance(cap, int) or cap < int(lun["CAPACITY"]): + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + lun["CAPACITY"] = str(cap) + return _ok() + + @route("DELETE", _D + r"/lun/(?P\d+)") + def delete_lun(self, req, dev, lid): + self._auth(req, dev) + self._lun(lid) + if any(lid in g["luns"] for g in self.lungroups.values()): + raise _fail(IN_USE, "The LUN is in a LUN group.") + del self.luns[lid] + return _ok() + + @route("GET", _D + "/lun/associate") + def lun_associate(self, req, dev): + self._auth(req, dev) + if req.q("ASSOCIATEOBJTYPE") != "21": + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + hid = req.q("ASSOCIATEOBJID") or "" + if hid not in self.hosts: + raise _fail(HOST_NOT_EXIST, "The host does not exist.") + rows = [] + for (h, lid), n in sorted(self.host_lun_ids.items()): + if h == hid: + row = dict(self.luns[lid]) + row["ASSOCIATEMETADATA"] = json.dumps({"HostLUNID": n, + "mappingviewId": self._view_of(hid)}) + rows.append(row) + return _ok(rows) + + # -- hosts ---------------------------------------------------------------------------- + + def _host(self, hid: str) -> dict[str, Any]: + if hid not in self.hosts: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + return self.hosts[hid] + + @route("GET", _D + "/host") + def get_hosts(self, req, dev): + self._auth(req, dev) + name = _filter_name(req) + return _ok([h for h in self.hosts.values() if name is None or h["NAME"] == name]) + + @route("POST", _D + "/host") + def post_host(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + name = b.get("NAME") + if not name or len(name) > 31: + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if any(h["NAME"] == name for h in self.hosts.values()): + raise _fail(OBJECT_NAME_EXISTS, "The name already exists.") + hid = str(self.next_id()) + self.hosts[hid] = {"ID": hid, "NAME": name, "TYPE": "21", + "OPERATIONSYSTEM": str(b.get("OPERATIONSYSTEM", 0)), + "ISADD2HOSTGROUP": "false", "HEALTHSTATUS": "1", + "RUNNINGSTATUS": "1", "DESCRIPTION": b.get("DESCRIPTION", "")} + return _ok(self.hosts[hid]) + + @route("GET", _D + r"/host/(?P\d+)") + def get_host(self, req, dev, hid): + self._auth(req, dev) + return _ok(self._host(hid)) + + @route("DELETE", _D + r"/host/(?P\d+)") + def delete_host(self, req, dev, hid): + self._auth(req, dev) + if hid not in self.hosts: + raise _fail(HOST_NOT_EXIST, "The host does not exist.") + if any(hid in g["hosts"] for g in self.hostgroups.values()): + raise _fail(IN_USE, "The host is in a host group.") + if any(i["PARENTID"] == hid for c in self.initiators.values() for i in c.values()): + raise _fail(IN_USE, "The host has initiators.") + del self.hosts[hid] + return _ok() + + @route("DELETE", _D + "/host/associate") + def host_leave_group(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + g = self.hostgroups.get(str(b.get("ID"))) + if g is None: + raise _fail(HOSTGROUP_NOT_EXIST, "The host group does not exist.") + hid = str(b.get("ASSOCIATEOBJID")) + if hid not in g["hosts"]: + raise _fail(HOST_NOT_IN_GROUP, "The host is not in the host group.") + g["hosts"].discard(hid) + self.hosts[hid]["ISADD2HOSTGROUP"] = "false" + self._refresh() + return _ok() + + # -- initiators ------------------------------------------------------------------------- + + def _ini_lookup(self, coll: str, iid: str) -> dict[str, Any] | None: + for k, v in self.initiators[coll].items(): + if k.lower() == iid.lower(): + return v + return None + + @route("GET", _D + r"/(?Piscsi_initiator|fc_initiator)") + def list_initiators(self, req, dev, coll): + self._auth(req, dev) + parent = req.q("PARENTID") + return _ok([i for i in self.initiators[coll].values() + if parent is None or i["PARENTID"] == parent]) + + @route("POST", _D + r"/(?Piscsi_initiator|fc_initiator)") + def add_initiator(self, req, dev, coll): + self._auth(req, dev) + b = req.json() or {} + iid = str(b.get("ID") or "") + if coll == "fc_initiator" and (len(iid) != 16 or any( + c not in "0123456789abcdef" for c in iid)): + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if not iid: + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if self._ini_lookup(coll, iid) is not None: + raise _fail(OBJECT_ID_NOT_UNIQUE, "The ID already exists.") + self.initiators[coll][iid] = { + "ID": iid, "TYPE": "222" if coll == "iscsi_initiator" else "223", + "ISFREE": "true", "PARENTID": "", "PARENTNAME": "", "PARENTTYPE": "", + "RUNNINGSTATUS": "28", "HEALTHSTATUS": "1", "MULTIPATHTYPE": "0"} + return _ok(self.initiators[coll][iid]) + + @route("GET", _D + r"/(?Piscsi_initiator|fc_initiator)/(?P[^/]+)") + def get_initiator(self, req, dev, coll, iid): + self._auth(req, dev) + ini = self._ini_lookup(coll, iid) + if ini is None: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + return _ok(ini) + + @route("PUT", _D + r"/(?Piscsi_initiator|fc_initiator)/(?P[^/]+)") + def put_initiator(self, req, dev, coll, iid): + self._auth(req, dev) + b = req.json() or {} + if iid in ("remove_iscsi_from_host", "remove_fc_from_host"): + ini = self._ini_lookup(coll, str(b.get("ID") or "")) + if ini is None: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + ini.update(ISFREE="true", PARENTID="", PARENTNAME="", PARENTTYPE="") + return _ok() + ini = self._ini_lookup(coll, iid) + if ini is None: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + if "PARENTID" in b: + hid = str(b["PARENTID"]) + host = self._host(hid) + if str(b.get("PARENTTYPE")) != "21": + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if ini["ISFREE"] == "false" and ini["PARENTID"] != hid: + raise _fail(ALREADY_ASSOCIATED, "The initiator has been added to another host.") + ini.update(ISFREE="false", PARENTID=hid, PARENTNAME=host["NAME"], PARENTTYPE="21") + return _ok() + + # -- groups and views --------------------------------------------------------------------- + + def _create_named(self, store: dict[str, dict[str, Any]], req: Request, + obj_type: str, **sets: set) -> Reply: + b = req.json() or {} + name = b.get("NAME") + if not name or len(name) > 31: + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if any(o["NAME"] == name for o in store.values()): + raise _fail(OBJECT_NAME_EXISTS, "The name already exists.") + oid = str(self.next_id()) + store[oid] = {"ID": oid, "NAME": name, "TYPE": obj_type, "DESCRIPTION": "", **sets} + return _ok(self._public(store[oid])) + + @staticmethod + def _public(obj: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in obj.items() if not isinstance(v, set)} + + def _list_named(self, store, req) -> Reply: + name = _filter_name(req) + return _ok([self._public(o) for o in store.values() if name is None or o["NAME"] == name]) + + @route("GET", _D + "/hostgroup") + def get_hostgroups(self, req, dev): + self._auth(req, dev) + return self._list_named(self.hostgroups, req) + + @route("POST", _D + "/hostgroup") + def post_hostgroup(self, req, dev): + self._auth(req, dev) + return self._create_named(self.hostgroups, req, "14", hosts=set()) + + @route("DELETE", _D + r"/hostgroup/(?P\d+)") + def delete_hostgroup(self, req, dev, gid): + self._auth(req, dev) + g = self.hostgroups.get(gid) + if g is None: + raise _fail(HOSTGROUP_NOT_EXIST, "The host group does not exist.") + if g["hosts"] or any(gid in v["hostgroups"] for v in self.views.values()): + raise _fail(IN_USE, "The host group is in use.") + del self.hostgroups[gid] + return _ok() + + @route("POST", _D + "/hostgroup/associate") + def hostgroup_add_host(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + g = self.hostgroups.get(str(b.get("ID"))) + if g is None: + raise _fail(HOSTGROUP_NOT_EXIST, "The host group does not exist.") + hid = str(b.get("ASSOCIATEOBJID")) + if hid not in self.hosts: + raise _fail(HOST_NOT_EXIST, "The host does not exist.") + if hid in g["hosts"]: + raise _fail(HOST_ALREADY_IN_GROUP, "The host already exists in the host group.") + if any(hid in o["hosts"] for o in self.hostgroups.values()): + raise _fail(ALREADY_ASSOCIATED, "The host already belongs to another host group.") + g["hosts"].add(hid) + self.hosts[hid]["ISADD2HOSTGROUP"] = "true" + self._refresh() + return _ok() + + @route("GET", _D + "/hostgroup/associate") + def hostgroups_of(self, req, dev): + self._auth(req, dev) + t, oid = req.q("ASSOCIATEOBJTYPE"), req.q("ASSOCIATEOBJID") or "" + if t == "21": + return _ok([self._public(g) for g in self.hostgroups.values() if oid in g["hosts"]]) + if t == "245": + v = self.views.get(oid) + return _ok([self._public(self.hostgroups[g]) for g in v["hostgroups"]] if v else []) + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + + @route("GET", _D + "/lungroup") + def get_lungroups(self, req, dev): + self._auth(req, dev) + return self._list_named(self.lungroups, req) + + @route("POST", _D + "/lungroup") + def post_lungroup(self, req, dev): + self._auth(req, dev) + return self._create_named(self.lungroups, req, "256", luns=set()) + + @route("DELETE", _D + r"/lungroup/(?P\d+)") + def delete_lungroup(self, req, dev, gid): + self._auth(req, dev) + g = self.lungroups.get(gid) + if g is None: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + if g["luns"] or any(gid in v["lungroups"] for v in self.views.values()): + raise _fail(IN_USE, "The LUN group is in use.") + del self.lungroups[gid] + return _ok() + + @route("POST", _D + "/lungroup/associate") + def lungroup_add_lun(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + g = self.lungroups.get(str(b.get("ID"))) + if g is None: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + if str(b.get("ASSOCIATEOBJTYPE")) != "11": + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + lid = str(b.get("ASSOCIATEOBJID")) + self._lun(lid) + if lid in g["luns"]: + raise _fail(LUN_ALREADY_IN_GROUP, "The LUN is already in the LUN group.") + g["luns"].add(lid) + self.luns[lid]["ISADD2LUNGROUP"] = "true" + self._refresh() + return _ok() + + @route("DELETE", _D + "/lungroup/associate") + def lungroup_remove_lun(self, req, dev): + self._auth(req, dev) + b = req.json() or {} + g = self.lungroups.get(str(b.get("ID"))) + lid = str(b.get("ASSOCIATEOBJID")) + if g is None or lid not in g["luns"]: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + g["luns"].discard(lid) + if lid in self.luns and not any(lid in o["luns"] for o in self.lungroups.values()): + self.luns[lid]["ISADD2LUNGROUP"] = "false" + self._refresh() + return _ok() + + @route("GET", _D + "/lungroup/associate") + def lungroups_of(self, req, dev): + self._auth(req, dev) + if req.q("ASSOCIATEOBJTYPE") != "11": + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + lid = req.q("ASSOCIATEOBJID") or "" + self._lun(lid) + return _ok([self._public(g) for g in self.lungroups.values() if lid in g["luns"]]) + + @route("GET", _D + "/mappingview") + def get_views(self, req, dev): + self._auth(req, dev) + return self._list_named(self.views, req) + + @route("POST", _D + "/mappingview") + def post_view(self, req, dev): + self._auth(req, dev) + return self._create_named(self.views, req, "245", hostgroups=set(), lungroups=set()) + + @route("DELETE", _D + r"/mappingview/(?P\d+)") + def delete_view(self, req, dev, vid): + self._auth(req, dev) + v = self.views.get(vid) + if v is None: + raise _fail(MAPPINGVIEW_NOT_EXIST, "The mapping view does not exist.") + if v["hostgroups"] or v["lungroups"]: + raise _fail(IN_USE, "The mapping view is in use.") + del self.views[vid] + return _ok() + + @route("PUT", _D + r"/mappingview/(?Pcreate_associate|remove_associate)") + def view_associate(self, req, dev, action): + self._auth(req, dev) + b = req.json() or {} + v = self.views.get(str(b.get("ID"))) + if v is None: + raise _fail(MAPPINGVIEW_NOT_EXIST, "The mapping view does not exist.") + t, oid = str(b.get("ASSOCIATEOBJTYPE")), str(b.get("ASSOCIATEOBJID")) + if t == "14": + key, store, already, missing = ("hostgroups", self.hostgroups, + HOSTGROUP_ALREADY_IN_VIEW, HOSTGROUP_NOT_IN_VIEW) + elif t == "256": + key, store, already, missing = ("lungroups", self.lungroups, + LUNGROUP_ALREADY_IN_VIEW, LUNGROUP_NOT_IN_VIEW) + else: + raise _fail(PARAMETER_INCORRECT, "The entered parameter is incorrect.") + if oid not in store: + raise _fail(OBJECT_NOT_EXIST, "The object does not exist.") + if action == "create_associate": + if oid in v[key]: + raise _fail(already, "The group is already in the mapping view.") + if v[key]: + raise _fail(PARAMETER_INCORRECT, "A mapping view holds one group of each kind.") + v[key].add(oid) + else: + if oid not in v[key]: + raise _fail(missing, "The group is not in the mapping view.") + v[key].discard(oid) + self._refresh() + return _ok() + + # -- visibility ------------------------------------------------------------------------------- + + def _view_of(self, hid: str) -> str: + for v in self.views.values(): + if any(hid in self.hostgroups[g]["hosts"] for g in v["hostgroups"]): + return v["ID"] + return "" + + def _refresh(self) -> None: + """Recompute which LUNs each host sees; keep existing host LUN ids.""" + visible: set[tuple[str, str]] = set() + for v in self.views.values(): + hosts = {h for g in v["hostgroups"] for h in self.hostgroups[g]["hosts"]} + luns = {lun for g in v["lungroups"] for lun in self.lungroups[g]["luns"]} + visible |= {(h, lun) for h in hosts for lun in luns} + self.host_lun_ids = {k: n for k, n in self.host_lun_ids.items() if k in visible} + for h, lun in sorted(visible - set(self.host_lun_ids)): + used = {n for (hh, _), n in self.host_lun_ids.items() if hh == h} + self.host_lun_ids[(h, lun)] = next(n for n in range(1, 4096) if n not in used) + + +def contract_setup() -> ContractSetup: + fake = FakeHuaweiDorado(username="admin", password="Admin@storage1") + return ContractSetup( + fake=fake, + config=make_config("huawei-dorado", "https://dorado.fake.example:8088", + storage_pool="StoragePool001"), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("Admin@storage1")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py b/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py new file mode 100644 index 0000000000..609726468c --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py @@ -0,0 +1,395 @@ +"""Fake IBM Storage Virtualize REST API (the subset the adapter uses). + +Mirrors the real API's shape: every command is ``POST /rest[/v1]/[/]`` +with CLI flags as JSON; failures are HTTP 500 with the CLI's ``CMMVCnnnnE`` +message as plain text; an unknown/expired token is HTTP 403. Quirks kept: + +* values are strings (ids, capacities, SCSI ids); +* ``capacity`` is human-readable (``10.00GB``) unless ``bytes`` is passed; +* detail views (``lsvdisk ``) are a list: the volume, then its copies; +* a one-row ``lsvdiskhostmap`` comes back as a bare dict; +* object ids are the lowest free integer, so a deleted volume's id is reused; +* ``mkvdiskhostmap`` to a second host needs ``force`` (CMMVC6071E); +* FC WWPNs that are not logged in need ``force`` on mkhost/addhostport. +""" +from __future__ import annotations + +import secrets +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_ROOT = r"/rest(?:/v1)?" + +E_NOT_EXIST = "CMMVC5753E The specified object does not exist or is not a suitable candidate." +E_NAME_RULES = ("CMMVC5754E The specified object does not exist, or the name supplied " + "does not meet the naming rules.") +E_IN_REQUEST = "CMMVC5842E The action failed because an object that was in the request does not exist." +E_EXISTS = "CMMVC6035E The action failed as the object already exists." +E_IQN = ("CMMVC6581E The command has failed because the maximum number of allowed iSCSI " + "qualified names (IQNs) has been reached, or the IQN is already assigned or is not valid.") +E_WWPN = ("CMMVC5867E The action failed because the worldwide port name is already " + "assigned or is not valid.") +E_ALREADY_MAPPED = ("CMMVC5878E The virtual disk (VDisk)-to-host mapping was not created " + "because this VDisk is already mapped to this host.") +E_MULTI_HOST = ("CMMVC6071E The VDisk-to-host mapping was not created because the VDisk is " + "already mapped to a host.") +E_VDISK_MAPPED = ("CMMVC5840E The virtual disk (VDisk) was not deleted because it is mapped " + "to a host or because it is part of a FlashCopy or Remote Copy mapping, " + "or is involved in an image mode migrate.") +E_HOST_MAPPED = ("CMMVC5871E The action failed because one or more of the configured port " + "names is in a mapping.") +E_BAD_NAME = ("CMMVC6527E The name that you have entered is not valid. The name can contain " + "letters, numbers, spaces, periods, dashes, and underscores. The name must " + "begin with a letter or an underscore.") +E_PARAM = "CMMVC5707E Required parameters are missing." +E_SHRINK = "CMMVC5709E [-size] is not a supported parameter." + + +def _cli_error(text: str) -> FakeError: + return FakeError(500, text.encode(), {"Content-Type": "text/plain"}) + + +def _human(size: int) -> str: + for unit, div in (("TB", 1 << 40), ("GB", 1 << 30), ("MB", 1 << 20)): + if size >= div: + return f"{size / div:.2f}{unit}" + return f"{size}B" + + +def _units(unit: str | None) -> int: + return 1 << (10 * ["b", "kb", "mb", "gb", "tb", "pb"].index((unit or "mb").lower())) + + +def _valid_name(name: str) -> bool: + return bool(name) and len(name) <= 63 and (name[0].isalpha() or name[0] == "_") and all( + c.isalnum() or c in "._- " for c in name) + + +class FakeIbmFlashSystem(FakeArray): + vendor = "ibm-flashsystem" + + def __init__(self, username: str = "superuser", password: str = "passw0rd", + pools: tuple[str, ...] = ("Pool0",)): + super().__init__() + self.username = username + self.password = password + self.pools = {name: str(i) for i, name in enumerate(pools)} + self.tokens: set[str] = set() + self.vdisks: dict[str, dict[str, Any]] = {} # id -> vdisk + self.hosts: dict[str, dict[str, Any]] = {} # name -> host + self.maps: list[dict[str, Any]] = [] # {vdisk_id, host, scsi} + self.commands: list[tuple[str, str | None, dict[str, Any]]] = [] + + # -- fake controls ------------------------------------------------------------ + + def expire_sessions(self) -> None: + self.tokens.clear() + + def set_credential(self, **fields: str) -> None: + self.username = fields["username"] + self.password = fields["password"] + + @staticmethod + def _free_id(used) -> str: + n = 0 + while str(n) in used: + n += 1 + return str(n) + + def add_volume(self, name: str, size: int, thin: bool = False) -> str: + vid = self._free_id(self.vdisks) + self.vdisks[vid] = {"id": vid, "name": name, "capacity": size, + "vdisk_UID": "600507680C8" + secrets.token_hex(11).upper()[:21], + "mdisk_grp_name": next(iter(self.pools)), "IO_group_name": "io_grp0", + "thin": thin} + return vid + + # -- dispatch ------------------------------------------------------------------- + + @route("POST", _ROOT + "/auth") + def auth(self, req: Request): + if (req.headers.get("X-Auth-Username") != self.username + or req.headers.get("X-Auth-Password") != self.password): + raise FakeError(403, {"code": 403, "message": "Forbidden"}) + token = secrets.token_hex(16) + self.tokens.add(token) + return Reply(200, {"token": token}) + + @route("POST", _ROOT + r"/(?P[a-z]+)(?:/(?P[^/]+))?") + def command(self, req: Request, cmd: str, obj: str | None = None): + if cmd == "auth": # pragma: no cover - matched above + return self.auth(req) + if req.headers.get("X-Auth-Token") not in self.tokens: + raise FakeError(403, {"code": 403, "message": "Forbidden"}) + opts = req.json() or {} + self.commands.append((cmd, obj, opts)) + handler = getattr(self, f"_c_{cmd}", None) + if handler is None: + raise FakeError(404, "Not Found") + result = handler(obj, opts) + return Reply(200, result) + + # -- lookups ---------------------------------------------------------------------- + + def _vdisk(self, ref: str | None) -> dict[str, Any]: + if ref in self.vdisks: + return self.vdisks[ref] + for v in self.vdisks.values(): + if v["name"] == ref: + return v + raise _cli_error(E_NOT_EXIST) + + def _host(self, ref: str | None) -> dict[str, Any]: + if ref in self.hosts: + return self.hosts[ref] + for h in self.hosts.values(): + if h["id"] == ref: + return h + raise _cli_error(E_NOT_EXIST) + + def _vdisk_row(self, v: dict[str, Any], opts: dict[str, Any]) -> dict[str, Any]: + cap = str(v["capacity"]) if opts.get("bytes") else _human(v["capacity"]) + return {"id": v["id"], "name": v["name"], "IO_group_id": "0", + "IO_group_name": v["IO_group_name"], "status": "online", + "mdisk_grp_id": self.pools[v["mdisk_grp_name"]], + "mdisk_grp_name": v["mdisk_grp_name"], "capacity": cap, "type": "striped", + "FC_id": "", "FC_name": "", "RC_id": "", "RC_name": "", + "vdisk_UID": v["vdisk_UID"], "fc_map_count": "0", "copy_count": "1", + "fast_write_state": "empty", "se_copy_count": "1" if v["thin"] else "0", + "RC_change": "no", "compressed_copy_count": "0", "parent_mdisk_grp_id": "0", + "parent_mdisk_grp_name": v["mdisk_grp_name"], "formatting": "no", + "encrypt": "no", "volume_id": v["id"], "volume_name": v["name"], + "function": ""} + + # -- system ------------------------------------------------------------------------- + + def _c_lssystem(self, obj, opts): + return {"id": "000002042140A0B2", "name": "fake-fs7300", "location": "local", + "code_level": "8.6.0.4 (build 169.12.2403211122000)", + "product_name": "IBM FlashSystem 7300"} + + # -- volumes -------------------------------------------------------------------------- + + def _c_lsvdisk(self, obj, opts): + if obj is not None: + v = self._vdisk(obj) + row = self._vdisk_row(v, opts) + cap = row["capacity"] + copy = {"copy_id": "0", "status": "online", "sync": "yes", "primary": "yes", + "mdisk_grp_name": v["mdisk_grp_name"], "type": "striped", + "capacity": cap, "real_capacity": cap if not v["thin"] else "0.00MB", + "autoexpand": "on" if v["thin"] else "", "se_copy": "yes" if v["thin"] else "no"} + return [row, copy] + rows = [self._vdisk_row(v, opts) for v in self.vdisks.values()] + fv = opts.get("filtervalue") + if fv: + key, _, value = str(fv).partition("=") + rows = [r for r in rows if str(r.get(key)) == value] + return rows + + def _c_mkvolume(self, obj, opts): + name, pool = opts.get("name"), opts.get("pool") + if "size" not in opts or not pool: + raise _cli_error(E_PARAM) + if pool not in self.pools and pool not in self.pools.values(): + raise _cli_error(E_NAME_RULES) + if name is not None and not _valid_name(name): + raise _cli_error(E_BAD_NAME) + if any(v["name"] == name for v in self.vdisks.values()): + raise _cli_error(E_EXISTS) + size = int(opts["size"]) * _units(opts.get("unit")) + size = -(-size // 512) * 512 + vid = self.add_volume(name or "", size, thin=bool(opts.get("thin"))) + v = self.vdisks[vid] + if not name: + v["name"] = f"volume{vid}" + if pool in self.pools: + v["mdisk_grp_name"] = pool + if opts.get("iogrp") is not None: + v["IO_group_name"] = str(opts["iogrp"]) + return {"id": vid, "message": f"Volume, id [{vid}], successfully created"} + + def _c_expandvdisksize(self, obj, opts): + v = self._vdisk(obj) + if "size" not in opts: + raise _cli_error(E_PARAM) + delta = int(opts["size"]) * _units(opts.get("unit")) + if delta <= 0: + raise _cli_error(E_SHRINK) + v["capacity"] += -(-delta // 512) * 512 + + def _c_rmvolume(self, obj, opts): + v = self._vdisk(obj) + if any(m["vdisk_id"] == v["id"] for m in self.maps) and not opts.get("removehostmappings"): + raise _cli_error(E_VDISK_MAPPED) + self.maps = [m for m in self.maps if m["vdisk_id"] != v["id"]] + del self.vdisks[v["id"]] + + # -- hosts ------------------------------------------------------------------------------ + + def _owner(self, kind: str, value: str) -> str | None: + for h in self.hosts.values(): + if value.lower() in (p.lower() for p in h[kind]): + return h["name"] + return None + + def _parse_ports(self, opts: dict[str, Any]) -> dict[str, list[str]]: + ports: dict[str, list[str]] = {"iscsiname": [], "fcwwpn": [], "nqn": []} + if opts.get("iscsiname"): + ports["iscsiname"] = [p for p in str(opts["iscsiname"]).split(",") if p] + if opts.get("fcwwpn"): + wwpns = [p for p in str(opts["fcwwpn"]).split(":") if p] + for w in wwpns: + if len(w) != 16 or any(c not in "0123456789abcdefABCDEF" for c in w): + raise _cli_error(E_WWPN) + if not opts.get("force"): + # The fake has no fabric: no WWPN is ever logged in. + raise _cli_error(E_WWPN) + ports["fcwwpn"] = [w.upper() for w in wwpns] + if opts.get("nqn"): + ports["nqn"] = [p for p in str(opts["nqn"]).split(",") if p] + return ports + + def _claim(self, host: str | None, ports: dict[str, list[str]]) -> None: + errors = {"iscsiname": E_IQN, "fcwwpn": E_WWPN, "nqn": E_IQN} + for kind, values in ports.items(): + for value in values: + if self._owner(kind, value) is not None: # even when it is this host + raise _cli_error(errors[kind]) + + def _c_mkhost(self, obj, opts): + name = opts.get("name") + if not name: + raise _cli_error(E_PARAM) + if not _valid_name(name): + raise _cli_error(E_BAD_NAME) + if name in self.hosts: + raise _cli_error(E_EXISTS) + protocol = opts.get("protocol", "scsi") + if protocol not in ("scsi", "iscsi", "fcscsi", "tcpnvme", "rdmanvme", "fcnvme", "nvme"): + raise _cli_error(E_PARAM) + ports = self._parse_ports(opts) + if not any(ports.values()): + raise _cli_error(E_PARAM) + if ports["nqn"] and protocol not in ("tcpnvme", "rdmanvme", "fcnvme", "nvme"): + raise _cli_error(E_PARAM) + self._claim(None, ports) + hid = self._free_id({h["id"] for h in self.hosts.values()}) + self.hosts[name] = {"id": hid, "name": name, "protocol": protocol, + "iscsiname": ports["iscsiname"], "fcwwpn": ports["fcwwpn"], + "nqn": ports["nqn"]} + return {"id": hid, "message": f"Host, id [{hid}], successfully created"} + + def _c_addhostport(self, obj, opts): + h = self._host(obj) + ports = self._parse_ports(opts) + if not any(ports.values()): + raise _cli_error(E_PARAM) + self._claim(h["name"], ports) + for kind, values in ports.items(): + h[kind].extend(values) + + def _c_lshost(self, obj, opts): + if obj is None: + return [{"id": h["id"], "name": h["name"], "port_count": str( + len(h["iscsiname"]) + len(h["fcwwpn"]) + len(h["nqn"])), + "iogrp_count": "4", "status": "online", "protocol": h["protocol"]} + for h in self.hosts.values()] + h = self._host(obj) + nodes = ([{"WWPN": w, "node_logged_in_count": "0", "state": "offline"} + for w in h["fcwwpn"]] + + [{"iscsi_name": i, "node_logged_in_count": "0", "state": "offline"} + for i in h["iscsiname"]] + + [{"nqn": n, "node_logged_in_count": "0", "state": "offline"} + for n in h["nqn"]]) + return {"id": h["id"], "name": h["name"], "port_count": str(len(nodes)), + "type": "generic", "iogrp_count": "4", "status": "offline", + "protocol": h["protocol"], "host_cluster_id": "", "host_cluster_name": "", + "mapping_count": str(sum(1 for m in self.maps if m["host"] == h["name"])), + "nodes": nodes} + + def _c_rmhost(self, obj, opts): + h = self._host(obj) + if any(m["host"] == h["name"] for m in self.maps) and not opts.get("force"): + raise _cli_error(E_HOST_MAPPED) + self.maps = [m for m in self.maps if m["host"] != h["name"]] + del self.hosts[h["name"]] + + # -- mappings ------------------------------------------------------------------------------- + + def _map_row(self, m: dict[str, Any]) -> dict[str, Any]: + v, h = self.vdisks[m["vdisk_id"]], self.hosts[m["host"]] + return {"id": v["id"], "name": v["name"], "SCSI_id": str(m["scsi"]), + "host_id": h["id"], "host_name": h["name"], "vdisk_UID": v["vdisk_UID"], + "IO_group_id": "0", "IO_group_name": v["IO_group_name"], + "mapping_type": "private", "host_cluster_id": "", "host_cluster_name": "", + "protocol": h["protocol"]} + + def _c_lsvdiskhostmap(self, obj, opts): + v = self._vdisk(obj) + rows = [self._map_row(m) for m in self.maps if m["vdisk_id"] == v["id"]] + return rows[0] if len(rows) == 1 else rows # quirk: one row -> bare dict + + def _c_mkvdiskhostmap(self, obj, opts): + v = self._vdisk(obj) + if not opts.get("host"): + raise _cli_error(E_PARAM) + h = self._host(opts["host"]) + mine = [m for m in self.maps if m["vdisk_id"] == v["id"]] + if any(m["host"] == h["name"] for m in mine): + raise _cli_error(E_ALREADY_MAPPED) + if mine and not opts.get("force"): + raise _cli_error(E_MULTI_HOST) + used = {m["scsi"] for m in self.maps if m["host"] == h["name"]} + scsi = int(opts["scsi"]) if opts.get("scsi") is not None else next( + n for n in range(4096) if n not in used) + self.maps.append({"vdisk_id": v["id"], "host": h["name"], "scsi": scsi}) + return {"id": str(scsi), + "message": f"Virtual Disk to Host map, id [{scsi}], successfully created"} + + def _c_rmvdiskhostmap(self, obj, opts): + v = self._vdisk(obj) + if not opts.get("host"): + raise _cli_error(E_PARAM) + h = self._host(opts["host"]) + before = len(self.maps) + self.maps = [m for m in self.maps + if not (m["vdisk_id"] == v["id"] and m["host"] == h["name"])] + if len(self.maps) == before: + raise _cli_error(E_IN_REQUEST) + + +def contract_setup() -> ContractSetup: + fake = FakeIbmFlashSystem(username="superuser", password="passw0rd") + return ContractSetup( + fake=fake, + config=make_config("ibm-flashsystem", "https://flashsystem.fake.example:7443", + pool="Pool0"), + credential=Credential({"username": SecretStr("superuser"), + "password": SecretStr("passw0rd")}), + bad_credential=Credential({"username": SecretStr("superuser"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + Protocol.NVME_TCP: ( + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-a"),), + (Initiator(protocol=Protocol.NVME_TCP, + address="nqn.2014-08.org.nvmexpress:uuid:host-b"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_volume(name, size), + ) diff --git a/simplyblock_core/storage_arrays/testing/infinidat.py b/simplyblock_core/storage_arrays/testing/infinidat.py new file mode 100644 index 0000000000..595b75cab6 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/infinidat.py @@ -0,0 +1,315 @@ +"""Fake Infinidat InfiniBox REST API (``/api/rest``), the subset the adapter +uses. + +Quirks reproduced: every reply is a ``{"result", "error", "metadata"}`` +envelope; lists are paged and filtered with ``field=eq:value``; destructive +requests without ``?approved=true`` get **403** ``APPROVAL_REQUIRED``; +object ids are integers; volume ``serial`` is the NAA-6 identifier without +its leading ``6``; LUN numbers start at 1 per host. Error codes follow the +``OBJECT_NOT_FOUND`` / ``*_CONFLICT`` style of the real API but are our +reading of it, not a verified list. For higher fidelity run the contract +suite's live tier against Infinidat's ``infinisim``. +""" +from __future__ import annotations + +from typing import Any + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol, normalize_wwpn +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_R = "/api/rest" +_ID = r"(?P\d+)" +_SYSTEM_ID = "000004e2" + + +def _err(status: int, code: str, message: str) -> FakeError: + return FakeError(status, { + "result": None, + "error": {"code": code, "message": message, "is_remote": False, "data": None, + "reasons": [], "severity": "ERROR"}, + "metadata": {"ready": True}}) + + +def _ok(result: Any, status: int = 200, metadata: dict[str, Any] | None = None) -> Reply: + return Reply(status, {"result": result, "error": None, + "metadata": metadata or {"ready": True}}) + + +def _port_address(ptype: str, address: str) -> str: + if ptype == "FC": + try: + return normalize_wwpn(address) + except ValueError: + raise _err(400, "INVALID_PORT_ADDRESS", f"Invalid FC address {address}") from None + return address.lower() + + +class FakeInfiniBox(FakeArray): + vendor = "infinidat-infinibox" + + def __init__(self, username: str = "admin", password: str = "ibox-pass"): + super().__init__() + self.username = username + self.password = password + self.pools: dict[int, dict[str, Any]] = {} + self.volumes: dict[int, dict[str, Any]] = {} + self.hosts: dict[int, dict[str, Any]] = {} + self.luns: list[dict[str, Any]] = [] # {host_id, volume_id, lun, id} + self.add_pool("sb-pool") + + # -- fake controls ------------------------------------------------------ + + def set_credential(self, **fields: str) -> None: + self.username = fields.get("username", self.username) + self.password = fields["password"] + + def add_pool(self, name: str) -> int: + pid = self.next_id() + self.pools[pid] = {"id": pid, "name": name, "physical_capacity": 1 << 50, + "virtual_capacity": 1 << 52, "state": "NORMAL"} + return pid + + def add_volume(self, name: str, size: int, pool_id: int | None = None) -> int: + vid = self.next_id() + self.volumes[vid] = { + "id": vid, "name": name, "size": size, "used": 0, "allocated": 0, + "pool_id": pool_id or next(iter(self.pools)), "provtype": "THIN", + "type": "MASTER", "mapped": False, "write_protected": False, + "serial": f"742b0f{_SYSTEM_ID}{vid:017x}", "created_at": 1763600000000, + "parent_id": 0, "family_id": vid, "rmr_source": False, "rmr_target": False, + } + return vid + + # -- dispatch helpers --------------------------------------------------------- + + def _auth(self, req: Request) -> None: + if req.basic_auth() != (self.username, self.password): + raise _err(401, "UNAUTHORIZED", "Invalid credentials") + + @staticmethod + def _approved(req: Request, what: str) -> None: + if (req.q("approved") or "").lower() != "true": + raise _err(403, "APPROVAL_REQUIRED", f"{what} requires approval") + + def _page(self, req: Request, items: list[dict[str, Any]]) -> Reply: + for key, values in req.query.items(): + if key in ("page", "page_size", "approved", "fields", "sort"): + continue + raw = values[0] + op, _, value = raw.partition(":") if ":" in raw else ("eq", "", raw) + if op != "eq": + raise _err(400, "UNSUPPORTED_FILTER", f"fake: filter op {op}") + items = [i for i in items if str(i.get(key)) == value] + page = int(req.q("page") or 1) + size = int(req.q("page_size") or 50) + if not 1 <= size <= 1000: + raise _err(400, "BAD_PAGE_SIZE", "page_size must be 1..1000") + total = len(items) + pages = max(1, -(-total // size)) + chunk = items[(page - 1) * size: page * size] + return _ok(chunk, metadata={"ready": True, "number_of_objects": total, + "page_size": size, "pages_total": pages, "page": page}) + + def _vol(self, vid: int) -> dict[str, Any]: + if vid not in self.volumes: + raise _err(404, "VOLUME_NOT_FOUND", f"Volume {vid} not found") + return self.volumes[vid] + + def _host(self, hid: int) -> dict[str, Any]: + if hid not in self.hosts: + raise _err(404, "HOST_NOT_FOUND", f"Host {hid} not found") + return self.hosts[hid] + + def _host_view(self, h: dict[str, Any]) -> dict[str, Any]: + luns = [self._lun_view(m) for m in self.luns if m["host_id"] == h["id"]] + return {**h, "ports": [dict(p) for p in h["ports"]], "luns": luns} + + def _lun_view(self, m: dict[str, Any]) -> dict[str, Any]: + return {"id": m["id"], "lun": m["lun"], "volume_id": m["volume_id"], + "host_id": m["host_id"], "host_cluster_id": 0, "clustered": False} + + # -- system / pools -------------------------------------------------------------- + + @route("GET", _R + "/system") + def system(self, req): + self._auth(req) + return _ok({"name": "ibox-fake", "serial_number": 1250, "model_name": "F4304", + "version": "7.3.10.0", "system_type": "INFINIBOX"}) + + @route("GET", _R + "/pools") + def list_pools(self, req): + self._auth(req) + return self._page(req, list(self.pools.values())) + + # -- volumes -------------------------------------------------------------------- + + @route("GET", _R + "/volumes") + def list_volumes(self, req): + self._auth(req) + return self._page(req, list(self.volumes.values())) + + @route("POST", _R + "/volumes") + def create_volume(self, req): + self._auth(req) + body = req.json() or {} + name, size, pool_id = str(body.get("name") or ""), body.get("size"), body.get("pool_id") + if not isinstance(size, int) or size <= 0 or size % 512: + raise _err(400, "BAD_REQUEST", "size must be a positive multiple of 512 bytes") + if pool_id not in self.pools: + raise _err(404, "POOL_NOT_FOUND", f"Pool {pool_id} not found") + if body.get("provtype", "THIN") not in ("THIN", "THICK"): + raise _err(400, "BAD_REQUEST", "provtype must be THIN or THICK") + if any(v["name"] == name for v in self.volumes.values()): + raise _err(409, "VOLUME_NAME_CONFLICT", f"Volume name {name} already exists") + vid = self.add_volume(name, size, pool_id) + self.volumes[vid]["provtype"] = body.get("provtype", "THIN") + return _ok(self.volumes[vid], 201) + + @route("GET", _R + "/volumes/" + _ID) + def get_volume(self, req, oid): + self._auth(req) + return _ok(self._vol(int(oid))) + + @route("PUT", _R + "/volumes/" + _ID) + def update_volume(self, req, oid): + self._auth(req) + v = self._vol(int(oid)) + body = req.json() or {} + if "size" in body: + size = body["size"] + if not isinstance(size, int) or size <= 0 or size % 512: + raise _err(400, "BAD_REQUEST", "size must be a positive multiple of 512 bytes") + if size < v["size"]: + self._approved(req, "Shrinking a volume") + v["size"] = size + return _ok(v) + + @route("DELETE", _R + "/volumes/" + _ID) + def delete_volume(self, req, oid): + self._auth(req) + v = self._vol(int(oid)) + self._approved(req, "Deleting a volume") + if any(m["volume_id"] == v["id"] for m in self.luns): + raise _err(409, "VOLUME_IS_MAPPED", "Volume is mapped; unmap it first") + del self.volumes[v["id"]] + return _ok(v) + + @route("GET", _R + "/volumes/" + _ID + "/luns") + def volume_luns(self, req, oid): + self._auth(req) + vid = self._vol(int(oid))["id"] + return _ok([self._lun_view(m) for m in self.luns if m["volume_id"] == vid]) + + # -- hosts ------------------------------------------------------------------------ + + @route("GET", _R + "/hosts") + def list_hosts(self, req): + self._auth(req) + return self._page(req, [self._host_view(h) for h in self.hosts.values()]) + + @route("POST", _R + "/hosts") + def create_host(self, req): + self._auth(req) + name = (req.json() or {}).get("name") + if not name: + raise _err(400, "BAD_REQUEST", "name is required") + if any(h["name"] == name for h in self.hosts.values()): + raise _err(409, "HOST_NAME_CONFLICT", f"Host name {name} already exists") + hid = self.next_id() + self.hosts[hid] = {"id": hid, "name": name, "ports": [], "security_method": "NONE", + "host_type": "LINUX", "host_cluster_id": 0} + return _ok(self._host_view(self.hosts[hid]), 201) + + @route("DELETE", _R + "/hosts/" + _ID) + def delete_host(self, req, oid): + self._auth(req) + h = self._host(int(oid)) + self._approved(req, "Deleting a host") + if any(m["host_id"] == h["id"] for m in self.luns): + raise _err(409, "HOST_HAS_LUNS", "Host has mapped volumes") + del self.hosts[h["id"]] + return _ok(self._host_view(h)) + + @route("POST", _R + "/hosts/" + _ID + "/ports") + def add_port(self, req, oid): + self._auth(req) + h = self._host(int(oid)) + body = req.json() or {} + ptype = str(body.get("type", "")).upper() + if ptype not in ("ISCSI", "FC"): + raise _err(400, "BAD_REQUEST", f"Unsupported port type {ptype}") + address = _port_address(ptype, str(body.get("address", ""))) + for other in self.hosts.values(): + for p in other["ports"]: + if p["type"] == ptype and p["address"] == address: + if other["id"] == h["id"]: + raise _err(409, "PORT_ALREADY_EXISTS", + f"Port {address} already belongs to this host") + raise _err(409, "PORT_ALREADY_BELONGS_TO_HOST", + f"Port {address} belongs to host {other['name']}") + port = {"type": ptype, "address": address, "host_id": h["id"]} + h["ports"].append(port) + return _ok(port, 201) + + @route("POST", _R + "/hosts/" + _ID + "/luns") + def map_lun(self, req, oid): + self._auth(req) + h = self._host(int(oid)) + vid = (req.json() or {}).get("volume_id") + if not isinstance(vid, int): + raise _err(400, "BAD_REQUEST", "volume_id must be an integer") + self._vol(vid) + if any(m["host_id"] == h["id"] and m["volume_id"] == vid for m in self.luns): + raise _err(409, "MAPPING_ALREADY_EXISTS", "Volume already mapped to host") + used = {m["lun"] for m in self.luns if m["host_id"] == h["id"]} + lun = next(n for n in range(1, 4096) if n not in used) + m = {"id": self.next_id(), "host_id": h["id"], "volume_id": vid, "lun": lun} + self.luns.append(m) + self.volumes[vid]["mapped"] = True + return _ok(self._lun_view(m), 201) + + @route("DELETE", _R + "/hosts/" + _ID + r"/luns/volume_id/(?P\d+)") + def unmap_lun(self, req, oid, vid): + self._auth(req) + h = self._host(int(oid)) + self._approved(req, "Unmapping a volume") + match = [m for m in self.luns if m["host_id"] == h["id"] and m["volume_id"] == int(vid)] + if not match: + raise _err(404, "LUN_NOT_FOUND", f"Volume {vid} is not mapped to host {h['id']}") + self.luns.remove(match[0]) + if int(vid) in self.volumes: + self.volumes[int(vid)]["mapped"] = any( + m["volume_id"] == int(vid) for m in self.luns) + return _ok(self._lun_view(match[0])) + + def not_routed(self, req: Request) -> Reply: + return Reply(404, {"result": None, "error": { + "code": "NOT_FOUND", "message": f"fake: no route for {req.method} {req.path}"}, + "metadata": {"ready": True}}) + + +def contract_setup() -> ContractSetup: + fake = FakeInfiniBox(username="admin", password="ibox-pass") + return ContractSetup( + fake=fake, + config=make_config("infinidat-infinibox", "https://ibox.fake.example", pool="sb-pool"), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("ibox-pass")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: str(fake.add_volume(name, size)), + ) diff --git a/simplyblock_core/storage_arrays/testing/netapp_ontap.py b/simplyblock_core/storage_arrays/testing/netapp_ontap.py new file mode 100644 index 0000000000..2ccc0b52c8 --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/netapp_ontap.py @@ -0,0 +1,832 @@ +"""Fake NetApp ONTAP 9.x REST API (the subset the adapter uses). + +Modelled behaviour: + +* HTTP basic auth on every request (``401`` + ONTAP error body otherwise). +* Collections answer ``{"records": [...], "num_records": n}``; query + parameters filter on (dotted, array-flattening) fields with ``|`` for OR + and ``*`` wildcards; ``fields=`` projects the records (key fields ``uuid`` + and ``name`` are always returned), so an adapter that forgets to ask for a + field does not get it. +* Errors: ``{"error": {"message": ..., "code": ...}}``; ``404`` for unknown + objects, ``409`` for duplicates, ``400`` for everything else. Numeric + codes are illustrative. +* FlexVol create/resize/delete are asynchronous: ``202`` with a job link; + the effect happens when the job runs, after ``job_polls`` polls of + ``/api/cluster/jobs/{uuid}``. +* Size rules: FlexVol >= 20 MiB (rounded up to 4 KiB); LUN/namespace must fit + in its FlexVol (usable = size minus snapshot reserve), cannot shrink, and a + namespace is a multiple of its block size. +* Name rules: FlexVol ``[A-Za-z_][A-Za-z0-9_]*``; LUN/namespace path + ``/vol//``. +* An initiator / host NQN may be in several igroups / subsystems (as on + ONTAP); a namespace can be mapped to one subsystem only; mapped + LUNs/namespaces/igroups/subsystems cannot be deleted. +""" +from __future__ import annotations + +import copy +import fnmatch +import re +import secrets +import string +import uuid as uuid_mod +from collections.abc import Callable +from typing import Any +from urllib.parse import unquote + +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential + +from ..base import Initiator, Protocol +from . import ContractSetup, FakeArray, FakeError, Reply, Request, make_config, route + +_U = r"(?P[0-9a-fA-F-]{36})" +_RESERVED = frozenset({"fields", "return_records", "return_timeout", "max_records", "order_by", + "allow_delete_while_mapped", "allow_delete_with_hosts"}) +_FLEXVOL_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,202}$") +_PATH_RE = re.compile(r"^/vol/(?P[^/]+)/(?P[A-Za-z0-9_.\-{}]{1,255})$") +_IGROUP_RE = re.compile(r"^[A-Za-z0-9_.:\-]{1,96}$") +_SUBSYS_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.\-]{0,95}$") +_WWPN_RE = re.compile(r"^([0-9a-f]{2}:){7}[0-9a-f]{2}$", re.IGNORECASE) +_SERIAL_CHARS = string.ascii_letters + string.digits + "-]?$" +MIN_VOLUME = 20 << 20 +MAX_LUN = 128 << 40 + + +def _err(status: int, message: str, code: str = "1", target: str | None = None) -> FakeError: + body: dict[str, Any] = {"error": {"message": message, "code": code}} + if target: + body["error"]["target"] = target + return FakeError(status, body) + + +def _not_found(what: str = "entry") -> FakeError: + return _err(404, f"{what} doesn't exist", "4") + + +def _resolve(obj: Any, parts: list[str]) -> list[Any]: + if not parts: + return obj if isinstance(obj, list) else [obj] + if isinstance(obj, list): + return [v for e in obj for v in _resolve(e, parts)] + if isinstance(obj, dict) and parts[0] in obj: + return _resolve(obj[parts[0]], parts[1:]) + return [] + + +def _matches(rec: dict[str, Any], key: str, pattern: str) -> bool: + values = _resolve(rec, key.split(".")) + texts = [str(v).lower() for v in values] # True -> "true", as in ONTAP queries + for p in pattern.split("|"): + if any(fnmatch.fnmatchcase(t, p.lower()) for t in texts): + return True + return False + + +def _project(obj: Any, paths: list[list[str]]) -> Any: + if any(not p for p in paths): + return copy.deepcopy(obj) + if isinstance(obj, list): + return [_project(e, paths) for e in obj] + if not isinstance(obj, dict): + return copy.deepcopy(obj) + out: dict[str, Any] = {} + groups: dict[str, list[list[str]]] = {} + for p in paths: + groups.setdefault(p[0], []).append(p[1:]) + for key, sub in groups.items(): + if key in obj: + out[key] = _project(obj[key], sub) + return out + + +class FakeNetAppOntap(FakeArray): + vendor = "netapp-ontap" + + def __init__(self, username: str = "admin", password: str = "netapp1!", + svm: str = "svm1"): + super().__init__() + self.username = username + self.password = password + self.svms: dict[str, str] = {svm: str(uuid_mod.uuid4())} + self.aggregates: list[dict[str, Any]] = [ + {"uuid": str(uuid_mod.uuid4()), "name": "aggr1", + "space": {"block_storage": {"available": 10 << 40}}}, + {"uuid": str(uuid_mod.uuid4()), "name": "aggr2", + "space": {"block_storage": {"available": 30 << 40}}}, + ] + self.volumes: dict[str, dict[str, Any]] = {} + self.luns: dict[str, dict[str, Any]] = {} + self.namespaces: dict[str, dict[str, Any]] = {} + self.igroups: dict[str, dict[str, Any]] = {} + self.lun_maps: list[dict[str, Any]] = [] + self.subsystems: dict[str, dict[str, Any]] = {} + self.subsystem_maps: list[dict[str, Any]] = [] + self.jobs: dict[str, dict[str, Any]] = {} + #: Number of job polls answering ``running`` before a job runs. + self.job_polls = 1 + #: When True, jobs never finish. + self.jobs_stuck = False + self._job_failures: list[tuple[str, str]] = [] + + # -- fake controls ------------------------------------------------------ + + def set_credential(self, **fields: str) -> None: + self.username = fields["username"] + self.password = fields["password"] + + def fail_next_job(self, message: str, code: str = "917504") -> None: + """Make the next job created end in ``failure`` with this message.""" + self._job_failures.append((message, code)) + + def svm_ref(self, name: str) -> dict[str, str]: + if name not in self.svms: + raise _err(404, f'SVM "{name}" not found', "2621462") + return {"name": name, "uuid": self.svms[name]} + + def add_flexvol(self, name: str, size: int, svm: str = "svm1", + snapshot_reserve: int = 0) -> str: + vid = str(uuid_mod.uuid4()) + self.volumes[vid] = { + "uuid": vid, "name": name, "svm": self.svm_ref(svm), "size": size, + "type": "rw", "style": "flexvol", "state": "online", + "guarantee": {"type": "none"}, + "space": {"snapshot": {"reserve_percent": snapshot_reserve}}, + "snapshot_policy": {"name": "default"}, + "aggregates": [{"name": "aggr1", "uuid": self.aggregates[0]["uuid"]}], + "comment": "", + } + return vid + + def add_lun(self, path: str, size: int, svm: str = "svm1") -> str: + vol = self._vol_of_path(path, svm) + lid = str(uuid_mod.uuid4()) + self.luns[lid] = { + "uuid": lid, "name": path, "svm": self.svm_ref(svm), "os_type": "linux", + "serial_number": "".join(secrets.choice(_SERIAL_CHARS) for _ in range(12)), + "space": {"size": size, "guarantee": {"requested": False, "reserved": False}}, + "location": {"logical_unit": path.rsplit("/", 1)[1], + "volume": {"name": vol["name"], "uuid": vol["uuid"]}}, + "status": {"mapped": False, "state": "online"}, + } + return lid + + def add_foreign_lun(self, name: str, size: int) -> str: + flex = name.replace("-", "_") + self.add_flexvol(flex, size * 2) + return self.add_lun(f"/vol/{flex}/{name}", size) + + # -- auth / plumbing ------------------------------------------------------ + + def _auth(self, req: Request) -> None: + if req.basic_auth() != (self.username, self.password): + raise _err(401, "User is not authorized.", "6") + + def _collection(self, req: Request, records: list[dict[str, Any]], + keys: tuple[str, ...] = ("uuid", "name")) -> Reply: + filters = {k: v[0] for k, v in req.query.items() if k not in _RESERVED and v} + found = [r for r in records if all(_matches(r, k, v) for k, v in filters.items())] + return Reply(200, {"records": [self._fields(r, req, keys) for r in found], + "num_records": len(found), + "_links": {"self": {"href": req.path}}}) + + @staticmethod + def _fields(rec: dict[str, Any], req: Request, keys: tuple[str, ...]) -> dict[str, Any]: + fields = req.q("fields") + if fields == "*": + return copy.deepcopy(rec) + paths = [[k] for k in keys if k in rec] + if fields: + paths += [f.strip().split(".") for f in fields.split(",") if f.strip()] + return _project(rec, paths) + + @staticmethod + def _created(req: Request, rec: dict[str, Any], location: str) -> Reply: + if req.q("return_records") == "true": + return Reply(201, {"num_records": 1, "records": [copy.deepcopy(rec)]}, + {"Location": location}) + return Reply(201, None, {"Location": location}) + + def _svm_of(self, body: dict[str, Any]) -> dict[str, str]: + svm = (body.get("svm") or {}).get("name") + if not svm: + raise _err(400, 'Field "svm.name" is required.', "2") + return self.svm_ref(svm) + + def _vol_of_path(self, path: str, svm: str) -> dict[str, Any]: + m = _PATH_RE.match(path or "") + if not m: + raise _err(400, f'Invalid path "{path}". Paths must be of the form ' + '"/vol//".', "5374082") + for v in self.volumes.values(): + if v["name"] == m["vol"] and v["svm"]["name"] == svm: + return v + raise _err(404, f'Volume "{m["vol"]}" in SVM "{svm}" does not exist.', "917927") + + @staticmethod + def _usable(vol: dict[str, Any]) -> int: + reserve = vol["space"]["snapshot"]["reserve_percent"] + return vol["size"] * (100 - reserve) // 100 + + # -- jobs ----------------------------------------------------------------- + + def _job(self, description: str, action: Callable[[], None]) -> Reply: + jid = str(uuid_mod.uuid4()) + failure = self._job_failures.pop(0) if self._job_failures else None + self.jobs[jid] = {"uuid": jid, "description": description, "state": "queued", + "message": "", "code": 0, "_polls": self.job_polls, + "_action": action, "_failure": failure} + href = f"/api/cluster/jobs/{jid}" + return Reply(202, {"job": {"uuid": jid, "_links": {"self": {"href": href}}}}) + + @route("GET", r"/api/cluster/jobs/" + _U) + def get_job(self, req, uuid): + self._auth(req) + job = self.jobs.get(uuid) + if job is None: + raise _not_found() + if job["state"] in ("queued", "running"): + if self.jobs_stuck or job["_polls"] > 0: + job["_polls"] -= 1 + job["state"] = "running" + job["message"] = "In progress" + elif job["_failure"]: + job["state"], job["message"] = "failure", job["_failure"][0] + job["code"] = int(job["_failure"][1]) + else: + try: + job["_action"]() + job["state"], job["message"] = "success", "success" + except FakeError as e: + err = e.reply.body["error"] + job["state"], job["message"] = "failure", err["message"] + job["code"] = int(err["code"]) + return Reply(200, {k: v for k, v in job.items() if not k.startswith("_")}) + + # -- cluster ---------------------------------------------------------------- + + @route("GET", "/api/cluster") + def cluster(self, req): + self._auth(req) + return Reply(200, {"name": "fake-ontap", "uuid": "1cd8a442-86d1-11e0-ae1c-123478563412", + "version": {"full": "NetApp Release 9.14.1: Mon Jan 01 00:00:00 UTC " + "2024", "generation": 9, "major": 14, + "minor": 1}}) + + @route("GET", "/api/cluster/nodes") + def nodes(self, req): + self._auth(req) + return self._collection(req, [ + {"uuid": "6ab1b4b3-0000-0000-0000-000000000001", "name": "fake-ontap-01", + "model": "AFF-A400", "serial_number": "721802000123"}, + {"uuid": "6ab1b4b3-0000-0000-0000-000000000002", "name": "fake-ontap-02", + "model": "AFF-A400", "serial_number": "721802000124"}, + ]) + + @route("GET", "/api/svm/svms") + def get_svms(self, req): + self._auth(req) + return self._collection(req, [{"uuid": u, "name": n} for n, u in self.svms.items()]) + + @route("GET", "/api/storage/aggregates") + def get_aggregates(self, req): + self._auth(req) + return self._collection(req, self.aggregates) + + # -- FlexVols --------------------------------------------------------------- + + @route("GET", "/api/storage/volumes") + def get_volumes(self, req): + self._auth(req) + return self._collection(req, list(self.volumes.values())) + + @route("GET", "/api/storage/volumes/" + _U) + def get_volume(self, req, uuid): + self._auth(req) + if uuid not in self.volumes: + raise _not_found() + return Reply(200, self._fields(self.volumes[uuid], req, ("uuid", "name")) if req.q( + "fields") else copy.deepcopy(self.volumes[uuid])) + + @route("POST", "/api/storage/volumes") + def post_volume(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + name = body.get("name", "") + if not _FLEXVOL_RE.match(name): + raise _err(400, f'Invalid volume name "{name}". Volume names must begin with a ' + 'letter or underscore and contain only letters, digits and ' + 'underscores.', "917888") + size = int(body.get("size", 0)) + if size < MIN_VOLUME: + raise _err(400, f"Volume size {size} is too small; minimum is {MIN_VOLUME}.", + "917534") + aggrs = body.get("aggregates") or [] + if len(aggrs) > 1: + # Several aggregates would create a FlexGroup: the adapter must not. + raise _err(400, "Specifying multiple aggregates creates a FlexGroup volume, " + "which is not supported for LUNs.", "917833") + known = {a["name"]: a for a in self.aggregates} + for a in aggrs: + if a.get("name") not in known: + raise _err(404, f'Aggregate "{a.get("name")}" does not exist.', "917927") + size = -(-size // 4096) * 4096 + + def create() -> None: + if any(v["name"] == name and v["svm"]["name"] == svm["name"] + for v in self.volumes.values()): + raise _err(409, f'Duplicate volume name "{name}".', "917535") + vid = self.add_flexvol(name, size, svm["name"], + int(((body.get("space") or {}).get("snapshot") or {}) + .get("reserve_percent", 5))) + vol = self.volumes[vid] + vol["guarantee"] = {"type": (body.get("guarantee") or {}).get("type", "volume")} + vol["snapshot_policy"] = {"name": (body.get("snapshot_policy") or {}) + .get("name", "default")} + vol["comment"] = body.get("comment", "") + if aggrs: + a = known[aggrs[0]["name"]] + vol["aggregates"] = [{"name": a["name"], "uuid": a["uuid"]}] + + return self._job(f"POST /api/storage/volumes/{name}", create) + + @route("PATCH", "/api/storage/volumes/" + _U) + def patch_volume(self, req, uuid): + self._auth(req) + if uuid not in self.volumes: + raise _not_found() + body = req.json() or {} + vol = self.volumes[uuid] + + def modify() -> None: + if "size" in body: + size = -(-int(body["size"]) // 4096) * 4096 + reserve = vol["space"]["snapshot"]["reserve_percent"] + biggest = max([lun["space"]["size"] for lun in self._luns_in(uuid)] or [0]) + if size < MIN_VOLUME or size * (100 - reserve) // 100 < biggest: + raise _err(400, "Cannot shrink the volume below the space used by its " + "LUNs or namespaces.", "917627") + vol["size"] = size + + return self._job(f"PATCH /api/storage/volumes/{uuid}", modify) + + def _luns_in(self, vol_uuid: str) -> list[dict[str, Any]]: + objs = list(self.luns.values()) + list(self.namespaces.values()) + return [o for o in objs if o["location"]["volume"]["uuid"] == vol_uuid] + + @route("DELETE", "/api/storage/volumes/" + _U) + def delete_volume(self, req, uuid): + self._auth(req) + if uuid not in self.volumes: + raise _not_found() + + def delete() -> None: + contained = self._luns_in(uuid) + if any(o["status"]["mapped"] for o in contained): + raise _err(400, "Volume contains mapped LUNs or namespaces.", "917536") + for o in contained: + self.luns.pop(o["uuid"], None) + self.namespaces.pop(o["uuid"], None) + del self.volumes[uuid] + + return self._job(f"DELETE /api/storage/volumes/{uuid}", delete) + + # -- LUNs and namespaces ------------------------------------------------------ + + def _check_size(self, vol: dict[str, Any], size: int, block: int = 512) -> None: + if size < 4096 or size > MAX_LUN: + raise _err(400, f"Size {size} is out of range (4096..{MAX_LUN}).", "5374855") + if size % block: + raise _err(400, f"Size {size} is not a multiple of the block size {block}.", + "72089652") + if size > self._usable(vol): + raise _err(400, f'Size {size} exceeds the space available in volume ' + f'"{vol["name"]}".', "5374857") + + @route("GET", "/api/storage/luns") + def get_luns(self, req): + self._auth(req) + return self._collection(req, list(self.luns.values())) + + @route("GET", "/api/storage/luns/" + _U) + def get_lun(self, req, uuid): + self._auth(req) + if uuid not in self.luns: + raise _not_found() + return Reply(200, copy.deepcopy(self.luns[uuid])) + + @route("POST", "/api/storage/luns") + def post_lun(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + path = body.get("name", "") + vol = self._vol_of_path(path, svm["name"]) + if not body.get("os_type"): + raise _err(400, 'Field "os_type" is required.', "2") + if any(o["name"] == path for o in list(self.luns.values()) + + list(self.namespaces.values())): + raise _err(409, f'A LUN or namespace already exists at path "{path}".', "5374863") + size = int((body.get("space") or {}).get("size", 0)) + self._check_size(vol, size) + lid = self.add_lun(path, size, svm["name"]) + lun = self.luns[lid] + lun["os_type"] = body["os_type"] + requested = bool(((body.get("space") or {}).get("guarantee") or {}).get("requested")) + lun["space"]["guarantee"] = {"requested": requested, "reserved": requested} + return self._created(req, lun, f"/api/storage/luns/{lid}") + + @route("PATCH", "/api/storage/luns/" + _U) + def patch_lun(self, req, uuid): + self._auth(req) + lun = self.luns.get(uuid) + if lun is None: + raise _not_found() + size = ((req.json() or {}).get("space") or {}).get("size") + if size is not None: + size = int(size) + if size < lun["space"]["size"]: + raise _err(400, "Reducing the size of a LUN might cause permanent data loss " + "or corruption and is not supported by the LUN REST API.", + "5374922") + self._check_size(self.volumes[lun["location"]["volume"]["uuid"]], size) + lun["space"]["size"] = size + return Reply(200, {}) + + @route("DELETE", "/api/storage/luns/" + _U) + def delete_lun(self, req, uuid): + self._auth(req) + lun = self.luns.get(uuid) + if lun is None: + raise _not_found() + if lun["status"]["mapped"] and req.q("allow_delete_while_mapped") != "true": + raise _err(400, "The LUN is mapped; unmap it or set allow_delete_while_mapped.", + "5374875") + self.lun_maps = [m for m in self.lun_maps if m["lun"]["uuid"] != uuid] + del self.luns[uuid] + return Reply(200, {}) + + @route("GET", "/api/storage/namespaces") + def get_namespaces(self, req): + self._auth(req) + return self._collection(req, list(self.namespaces.values())) + + @route("POST", "/api/storage/namespaces") + def post_namespace(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + path = body.get("name", "") + vol = self._vol_of_path(path, svm["name"]) + if not body.get("os_type"): + raise _err(400, 'Field "os_type" is required.', "2") + if any(o["name"] == path for o in list(self.luns.values()) + + list(self.namespaces.values())): + raise _err(409, f'A LUN or namespace already exists at path "{path}".', "72090019") + space = body.get("space") or {} + block = int(space.get("block_size", 4096)) + if block not in (512, 4096): + raise _err(400, "block_size must be 512 or 4096.", "72089653") + size = int(space.get("size", 0)) + self._check_size(vol, size, block) + nid = str(uuid_mod.uuid4()) + self.namespaces[nid] = { + "uuid": nid, "name": path, "svm": svm, "os_type": body["os_type"], + "space": {"size": size, "block_size": block}, + "location": {"namespace": path.rsplit("/", 1)[1], + "volume": {"name": vol["name"], "uuid": vol["uuid"]}}, + "status": {"mapped": False, "state": "online"}, + } + return self._created(req, self.namespaces[nid], f"/api/storage/namespaces/{nid}") + + @route("PATCH", "/api/storage/namespaces/" + _U) + def patch_namespace(self, req, uuid): + self._auth(req) + ns = self.namespaces.get(uuid) + if ns is None: + raise _not_found() + size = ((req.json() or {}).get("space") or {}).get("size") + if size is not None: + size = int(size) + if size < ns["space"]["size"]: + raise _err(400, "Shrinking a namespace is not supported.", "72090031") + self._check_size(self.volumes[ns["location"]["volume"]["uuid"]], size, + ns["space"]["block_size"]) + ns["space"]["size"] = size + return Reply(200, {}) + + @route("DELETE", "/api/storage/namespaces/" + _U) + def delete_namespace(self, req, uuid): + self._auth(req) + ns = self.namespaces.get(uuid) + if ns is None: + raise _not_found() + if ns["status"]["mapped"] and req.q("allow_delete_while_mapped") != "true": + raise _err(400, "The namespace is mapped to a subsystem.", "72090024") + self.subsystem_maps = [m for m in self.subsystem_maps if m["namespace"]["uuid"] != uuid] + del self.namespaces[uuid] + return Reply(200, {}) + + # -- igroups ------------------------------------------------------------------- + + def _check_initiator(self, group: dict[str, Any], name: str) -> None: + proto = group["protocol"] + is_iscsi = name.lower().startswith(("iqn.", "eui.", "naa.")) + is_fc = bool(_WWPN_RE.match(name)) + if not (is_iscsi or is_fc) or (proto == "iscsi" and not is_iscsi) or ( + proto == "fcp" and not is_fc): + raise _err(400, f'Initiator "{name}" is not valid for an igroup of protocol ' + f'"{proto}".', "1254324") + for g in self.igroups.values(): + if g is group or g["svm"]["name"] != group["svm"]["name"]: + continue + if name.lower() in (i["name"].lower() for i in g["initiators"]) and \ + g["os_type"] != group["os_type"]: + raise _err(400, f'Initiator "{name}" is in igroup "{g["name"]}" with a ' + "different os_type.", "1254323") + if name.lower() in (i["name"].lower() for i in group["initiators"]): + raise _err(409, f'Initiator "{name}" is already in igroup "{group["name"]}".', + "1254327") + + @route("GET", "/api/protocols/san/igroups") + def get_igroups(self, req): + self._auth(req) + return self._collection(req, list(self.igroups.values())) + + @route("POST", "/api/protocols/san/igroups") + def post_igroup(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + name = body.get("name", "") + if not _IGROUP_RE.match(name): + raise _err(400, f'Invalid igroup name "{name}".', "5374737") + if any(g["name"] == name and g["svm"]["name"] == svm["name"] + for g in self.igroups.values()): + raise _err(409, f'An igroup named "{name}" already exists.', "5374734") + proto = body.get("protocol", "mixed") + if proto not in ("iscsi", "fcp", "mixed"): + raise _err(400, f'Invalid protocol "{proto}".', "2") + gid = str(uuid_mod.uuid4()) + group = {"uuid": gid, "name": name, "svm": svm, "protocol": proto, + "os_type": body.get("os_type", "linux"), "initiators": []} + for i in body.get("initiators") or []: + self._check_initiator(group, i["name"]) + group["initiators"].append({"name": i["name"]}) + self.igroups[gid] = group + return self._created(req, group, f"/api/protocols/san/igroups/{gid}") + + @route("PATCH", "/api/protocols/san/igroups/" + _U) + def patch_igroup(self, req, uuid): + self._auth(req) + group = self.igroups.get(uuid) + if group is None: + raise _not_found() + proto = (req.json() or {}).get("protocol") + if proto is not None: + if proto not in ("iscsi", "fcp", "mixed"): + raise _err(400, f'Invalid protocol "{proto}".', "2") + group["protocol"] = proto + return Reply(200, {}) + + @route("POST", "/api/protocols/san/igroups/" + _U + "/initiators") + def post_igroup_initiators(self, req, uuid): + self._auth(req) + group = self.igroups.get(uuid) + if group is None: + raise _not_found() + body = req.json() or {} + records = body.get("records") or [body] + for r in records: + self._check_initiator(group, r["name"]) + for r in records: + group["initiators"].append({"name": r["name"]}) + return Reply(201, {}) + + @route("DELETE", "/api/protocols/san/igroups/" + _U) + def delete_igroup(self, req, uuid): + self._auth(req) + if uuid not in self.igroups: + raise _not_found() + if any(m["igroup"]["uuid"] == uuid for m in self.lun_maps) and \ + req.q("allow_delete_while_mapped") != "true": + raise _err(400, "The igroup is mapped to one or more LUNs.", "5374852") + del self.igroups[uuid] + return Reply(200, {}) + + # -- lun-maps ---------------------------------------------------------------------- + + @route("GET", "/api/protocols/san/lun-maps") + def get_lun_maps(self, req): + self._auth(req) + return self._collection(req, self.lun_maps, keys=("svm", "lun", "igroup")) + + def _find(self, table: dict[str, dict[str, Any]], ref: dict[str, Any], svm: str, + what: str) -> dict[str, Any]: + for obj in table.values(): + if obj["svm"]["name"] != svm: + continue + if ref.get("uuid") == obj["uuid"] or (ref.get("name") and + ref.get("name") == obj["name"]): + return obj + raise _err(404, f'{what} "{ref.get("name") or ref.get("uuid")}" does not exist.', "4") + + @route("POST", "/api/protocols/san/lun-maps") + def post_lun_map(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + lun = self._find(self.luns, body.get("lun") or {}, svm["name"], "LUN") + group = self._find(self.igroups, body.get("igroup") or {}, svm["name"], "igroup") + if any(m["lun"]["uuid"] == lun["uuid"] and m["igroup"]["uuid"] == group["uuid"] + for m in self.lun_maps): + raise _err(409, "The LUN is already mapped to this igroup.", "5374922") + used = {m["logical_unit_number"] for m in self.lun_maps + if m["igroup"]["uuid"] == group["uuid"]} + number = body.get("logical_unit_number") + if number is None: + number = next(n for n in range(4096) if n not in used) + elif number in used: + raise _err(409, f"LUN number {number} is in use in this igroup.", "5374921") + m = {"svm": svm, "lun": {"uuid": lun["uuid"], "name": lun["name"]}, + "igroup": {"uuid": group["uuid"], "name": group["name"]}, + "logical_unit_number": number} + self.lun_maps.append(m) + lun["status"]["mapped"] = True + return self._created(req, m, f"/api/protocols/san/lun-maps/{lun['uuid']}/" + f"{group['uuid']}") + + @route("DELETE", "/api/protocols/san/lun-maps/" + r"(?P[0-9a-fA-F-]{36})/" + + r"(?P[0-9a-fA-F-]{36})") + def delete_lun_map(self, req, lun, igroup): + self._auth(req) + before = len(self.lun_maps) + self.lun_maps = [m for m in self.lun_maps + if not (m["lun"]["uuid"] == lun and m["igroup"]["uuid"] == igroup)] + if len(self.lun_maps) == before: + raise _not_found() + if lun in self.luns: + self.luns[lun]["status"]["mapped"] = any( + m["lun"]["uuid"] == lun for m in self.lun_maps) + return Reply(200, {}) + + # -- NVMe subsystems ----------------------------------------------------------------- + + @route("GET", "/api/protocols/nvme/subsystems") + def get_subsystems(self, req): + self._auth(req) + return self._collection(req, list(self.subsystems.values())) + + @staticmethod + def _check_nqn(nqn: str) -> None: + if not nqn.startswith("nqn."): + raise _err(400, f'Invalid host NQN "{nqn}".', "72089705") + + @route("POST", "/api/protocols/nvme/subsystems") + def post_subsystem(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + name = body.get("name", "") + if not _SUBSYS_RE.match(name): + raise _err(400, f'Invalid subsystem name "{name}".', "72089771") + if not body.get("os_type"): + raise _err(400, 'Field "os_type" is required.', "2") + if any(s["name"] == name and s["svm"]["name"] == svm["name"] + for s in self.subsystems.values()): + raise _err(409, f'Subsystem "{name}" already exists.', "72089709") + hosts = [] + for h in body.get("hosts") or []: + self._check_nqn(h["nqn"]) + hosts.append({"nqn": h["nqn"]}) + sid = str(uuid_mod.uuid4()) + self.subsystems[sid] = {"uuid": sid, "name": name, "svm": svm, + "os_type": body["os_type"], "comment": body.get("comment", ""), + "hosts": hosts, + "target_nqn": f"nqn.1992-08.com.netapp:sn.{sid.replace('-', '')}" + f":subsystem.{name}"} + return self._created(req, self.subsystems[sid], f"/api/protocols/nvme/subsystems/{sid}") + + @route("POST", "/api/protocols/nvme/subsystems/" + _U + "/hosts") + def post_subsystem_hosts(self, req, uuid): + self._auth(req) + sub = self.subsystems.get(uuid) + if sub is None: + raise _not_found() + body = req.json() or {} + records = body.get("records") or [body] + have = {h["nqn"] for h in sub["hosts"]} + for r in records: + self._check_nqn(r["nqn"]) + if r["nqn"] in have: + raise _err(409, f'Host "{r["nqn"]}" is already in subsystem "{sub["name"]}".', + "72089708") + sub["hosts"].extend({"nqn": r["nqn"]} for r in records) + return Reply(201, {}) + + @route("DELETE", "/api/protocols/nvme/subsystems/" + _U + "/hosts/(?P[^/]+)") + def delete_subsystem_host(self, req, uuid, nqn): + self._auth(req) + sub = self.subsystems.get(uuid) + if sub is None: + raise _not_found() + nqn = unquote(nqn) + if nqn not in {h["nqn"] for h in sub["hosts"]}: + raise _not_found() + sub["hosts"] = [h for h in sub["hosts"] if h["nqn"] != nqn] + return Reply(200, {}) + + @route("DELETE", "/api/protocols/nvme/subsystems/" + _U) + def delete_subsystem(self, req, uuid): + self._auth(req) + sub = self.subsystems.get(uuid) + if sub is None: + raise _not_found() + if any(m["subsystem"]["uuid"] == uuid for m in self.subsystem_maps) and \ + req.q("allow_delete_while_mapped") != "true": + raise _err(400, "The subsystem has mapped namespaces.", "72090025") + if sub["hosts"] and req.q("allow_delete_with_hosts") != "true": + raise _err(400, "The subsystem has hosts; set allow_delete_with_hosts.", + "72090026") + for m in [m for m in self.subsystem_maps if m["subsystem"]["uuid"] == uuid]: + self.namespaces[m["namespace"]["uuid"]]["status"]["mapped"] = False + self.subsystem_maps = [m for m in self.subsystem_maps if m["subsystem"]["uuid"] != uuid] + del self.subsystems[uuid] + return Reply(200, {}) + + # -- subsystem-maps -------------------------------------------------------------------- + + @route("GET", "/api/protocols/nvme/subsystem-maps") + def get_subsystem_maps(self, req): + self._auth(req) + return self._collection(req, self.subsystem_maps, keys=("svm", "namespace", "subsystem")) + + @route("POST", "/api/protocols/nvme/subsystem-maps") + def post_subsystem_map(self, req): + self._auth(req) + body = req.json() or {} + svm = self._svm_of(body) + ns = self._find(self.namespaces, body.get("namespace") or {}, svm["name"], "Namespace") + sub = self._find(self.subsystems, body.get("subsystem") or {}, svm["name"], + "Subsystem") + if any(m["namespace"]["uuid"] == ns["uuid"] for m in self.subsystem_maps): + raise _err(400, "The namespace is already mapped to a subsystem. A namespace " + "can be mapped to only one subsystem.", "72089771") + used = {int(m["nsid"].rstrip("h"), 16) for m in self.subsystem_maps + if m["subsystem"]["uuid"] == sub["uuid"]} + nsid = next(n for n in range(1, 4096) if n not in used) + m = {"svm": svm, "namespace": {"uuid": ns["uuid"], "name": ns["name"]}, + "subsystem": {"uuid": sub["uuid"], "name": sub["name"]}, + "nsid": f"{nsid:08x}h"} + self.subsystem_maps.append(m) + ns["status"]["mapped"] = True + return self._created(req, m, f"/api/protocols/nvme/subsystem-maps/{sub['uuid']}/" + f"{ns['uuid']}") + + @route("DELETE", "/api/protocols/nvme/subsystem-maps/" + r"(?P[0-9a-fA-F-]{36})/" + + r"(?P[0-9a-fA-F-]{36})") + def delete_subsystem_map(self, req, sub, ns): + self._auth(req) + before = len(self.subsystem_maps) + self.subsystem_maps = [m for m in self.subsystem_maps + if not (m["subsystem"]["uuid"] == sub + and m["namespace"]["uuid"] == ns)] + if len(self.subsystem_maps) == before: + raise _not_found() + if ns in self.namespaces: + self.namespaces[ns]["status"]["mapped"] = False + return Reply(200, {}) + + def not_routed(self, req: Request) -> Reply: + return Reply(404, {"error": {"message": f"API not found: {req.method} {req.path}", + "code": "3"}}) + + +def contract_setup(**options: Any) -> ContractSetup: + fake = FakeNetAppOntap(username="admin", password="netapp1!", svm="svm1") + return ContractSetup( + fake=fake, + config=make_config("netapp-ontap", "https://ontap.fake.example", + **{"svm": "svm1", **options}), + credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("netapp1!")}), + bad_credential=Credential({"username": SecretStr("admin"), + "password": SecretStr("wrong")}), + # Default protocol_family is scsi: the contract maps every volume to + # hosts of each listed protocol, so only SCSI initiators are listed. + # NVMe is covered by tests/unit/storage_arrays/test_netapp_ontap.py. + initiators={ + Protocol.ISCSI: ( + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a"),), + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-b"),), + ), + Protocol.FC: ( + (Initiator(protocol=Protocol.FC, address="21:00:00:24:FF:4C:1A:2B"),), + (Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),), + ), + }, + create_foreign_volume=lambda name, size: fake.add_foreign_lun(name, size), + ) diff --git a/tests/unit/storage_arrays/test_contract.py b/tests/unit/storage_arrays/test_contract.py index 6e36507a59..1e63e0fdc8 100644 --- a/tests/unit/storage_arrays/test_contract.py +++ b/tests/unit/storage_arrays/test_contract.py @@ -76,7 +76,7 @@ def test_create_get_find(harness, cleanup): assert vol.size_bytes == round_size(10 * GIB + 1, caps.size_granularity_bytes, caps.min_size_bytes) assert vol.identifier and vol.identifier == vol.identifier.lower() - assert vol.wwn is None or vol.wwn.startswith("naa.") + assert vol.wwn is None or vol.wwn.startswith(("naa.", "eui.")) assert vol.nguid is None or vol.nguid.startswith("eui.") got = harness.client.get_volume(vol.id) @@ -177,6 +177,11 @@ def test_existing_host_gets_missing_initiators(harness, cleanup): name = f"sbtest-grow-{uuid.uuid4().hex[:6]}" host = harness.client.ensure_host(HostSpec(name=name, initiators=a)) cleanup[1].append(host) + limit = harness.client.capabilities.max_initiators_per_host + if limit is not None and len(a + b) > limit: + with pytest.raises(ArrayError, match="at most"): + harness.client.ensure_host(HostSpec(name=name, initiators=a + b)) + return assert harness.client.ensure_host(HostSpec(name=name, initiators=a + b)) == host # b's initiators now belong to that host: claiming them elsewhere conflicts with pytest.raises(ArrayConflictError): @@ -196,6 +201,20 @@ def test_wrong_identifier_is_refused(harness, cleanup): assert harness.client.get_volume(vol.id) is not None +def test_volume_of_another_record_is_refused(harness, cleanup): + """A reused array id/WWN must not pass for the recorded volume: the + name derived from the recorded UUID pins it.""" + vol_uuid = _uuid() + vol = harness.client.create_volume(vol_uuid, GIB) + cleanup[0].append(vol.id) + with pytest.raises(ArrayConflictError, match="name does not match"): + harness.client.delete_volume(vol.id, vol.identifier, volume_uuid=_uuid()) + with pytest.raises(ArrayConflictError, match="name does not match"): + harness.client.resize_volume(vol.id, 4 * GIB, volume_uuid=_uuid()) + assert harness.client.delete_volume(vol.id, vol.identifier, volume_uuid=vol_uuid) + cleanup[0].remove(vol.id) + + def test_foreign_volume_is_refused(harness): if harness.live: pytest.skip("live tier never touches foreign volumes") @@ -236,14 +255,11 @@ def test_rotated_credential_is_picked_up(harness): from pydantic import SecretStr from simplyblock_core.credentials import Credential - old_secret_values = {harness.setup.credential.get(k).get_secret_value() - for k in harness.setup.credential.field_names} harness.provider.credential = Credential({k: SecretStr(v) for k, v in new.items()}) - # A basic-auth adapter keeps the old password in its session until it is - # rejected; either way the client must recover with one invalidation. + # The client still holds the old credential; the array rejects it, the + # client invalidates once, resolves the rotated secret and succeeds. harness.client.get_info() - assert harness.provider.invalidated <= 1 - assert old_secret_values # silence unused + assert harness.provider.invalidated == 1 def test_server_error_does_not_leak_credentials(harness): @@ -268,3 +284,19 @@ def test_unsupported_protocol_is_refused_before_calling_the_array(harness): with pytest.raises(ArrayError, match="does not support"): harness.client.ensure_host(HostSpec(name="sbtest-unsup", initiators=(Initiator(protocol=p, address=address),))) + + +def test_delete_host_with_mappings_is_refused(harness, cleanup): + """Deleting a host must never silently drop a live mapping.""" + protocol = _protocols(harness)[0] + vol = harness.client.create_volume(_uuid(), GIB) + cleanup[0].append(vol.id) + host = harness.client.ensure_host(_host(harness, protocol, 0)) + cleanup[1].append(host) + harness.client.map_volume(vol.id, host) + try: + harness.client.delete_host(host) + except ArrayConflictError: + pass # refused: the expected outcome on arrays that own host objects + # Either way (refused, or a no-op as for PowerFlex SDCs), the mapping survives. + assert [m.host_id for m in harness.client.list_mappings(vol.id)] == [host] diff --git a/tests/unit/storage_arrays/test_dell_powerflex.py b/tests/unit/storage_arrays/test_dell_powerflex.py new file mode 100644 index 0000000000..504cef40ea --- /dev/null +++ b/tests/unit/storage_arrays/test_dell_powerflex.py @@ -0,0 +1,189 @@ +"""Dell PowerFlex specifics beyond the shared contract.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + ArrayNotFoundError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport, make_config +from simplyblock_core.storage_arrays.testing.dell_powerflex import ( + FakePowerFlex, + contract_setup, +) + +from .conftest import StaticProvider + +GIB = 1 << 30 + + +def _client(setup, **options): + config = setup.config + if options: + config = make_config("dell-powerflex", config.endpoint, + **{**config.options, **options}) + return ArrayClient(config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + + +def _sdc(setup, which=0) -> Initiator: + return setup.initiators[Protocol.SDC][which][0] + + +@pytest.fixture(params=[3, 4], ids=["v3-token", "v4-oauth"]) +def setup(request): + return contract_setup(generation=request.param) + + +def test_auto_auth_uses_generation_specific_login(setup): + with _client(setup) as c: + info = c.get_info() + fake: FakePowerFlex = setup.fake + paths = [p for _, p in fake.requests] + assert paths[0] == "/rest/auth/login" + if fake.generation == 3: + assert "/api/login" in paths and "R3_6" in info.version + else: + assert "/api/login" not in paths and "R4_5" in info.version + # close() logged out + assert not fake.tokens and not fake.access_tokens + + +def test_pinned_token_auth_skips_oauth_probe(): + setup = contract_setup(generation=3) + with _client(setup, auth="token") as c: + c.get_info() + assert ("POST", "/rest/auth/login") not in setup.fake.requests + + +def test_pinned_oauth_on_3x_fails(): + setup = contract_setup(generation=3) + with _client(setup, auth="oauth") as c, pytest.raises(ArrayNotFoundError): + c.get_info() + + +def test_bad_options_rejected(): + setup = contract_setup() + for opts in ({"auth": "kerberos"}, {"sdc_naming": "maybe"}, {"bogus": 1}): + with _client(setup, **opts) as c, pytest.raises(ArrayError): + c.get_info() + config = make_config("dell-powerflex", "https://pf.example") + with ArrayClient(config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake)) as c, \ + pytest.raises(ArrayError, match="storage_pool_id"): + c.get_info() + + +def test_sizes_round_to_8_gib(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), 10 * GIB) + assert vol.size_bytes == 16 * GIB + raw = setup.fake.volumes[vol.id] + assert raw["sizeInKb"] == 16 * 1024 * 1024 + assert raw["volumeType"] == "ThinProvisioned" + grown = c.resize_volume(vol.id, 17 * GIB) + assert grown.size_bytes == 24 * GIB + assert vol.wwn is None and vol.nguid is None + assert vol.identifier == f"id.{vol.id}" + + +def test_not_found_500_is_none_but_other_500_raises(setup): + with _client(setup) as c: + assert c.get_volume("deadbeef00000001") is None + c.get_info() + setup.fake.fail_next(500, {"message": "MDM is busy", "httpStatusCode": 500, + "errorCode": 3}, path="Volume::") + with pytest.raises(ArrayRequestError, match="MDM is busy"): + c.get_volume("deadbeef00000001") + + +def test_duplicate_name_conflicts(setup): + setup.fake.add_volume("sb-taken", 8 * GIB) + with _client(setup) as c: + c.get_info() + adapter = c._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError): + adapter.create_volume("sb-taken", 8 * GIB) + + +def test_ensure_host_finds_preregistered_sdc_and_claims_name(setup): + with _client(setup) as c: + host = c.ensure_host(HostSpec(name="worker-1", initiators=(_sdc(setup),))) + assert setup.fake.sdcs[host]["name"] == "worker-1" + # GUID match is case-insensitive + lower = Initiator(protocol=Protocol.SDC, address=_sdc(setup).address.lower()) + assert c.ensure_host(HostSpec(name="worker-1", initiators=(lower,))) == host + with pytest.raises(ArrayConflictError, match="worker-1"): + c.ensure_host(HostSpec(name="worker-2", initiators=(_sdc(setup),))) + # delete_host leaves the SDC alone + c.delete_host(host) + assert host in setup.fake.sdcs + + +def test_unknown_sdc_guid_is_not_found(setup): + ghost = Initiator(protocol=Protocol.SDC, address=str(uuid.uuid4())) + with _client(setup) as c, pytest.raises(ArrayNotFoundError, match="install the SDC"): + c.ensure_host(HostSpec(name="worker-9", initiators=(ghost,))) + + +def test_multi_sdc_host_is_refused_by_the_client(setup): + spec = HostSpec(name="worker-1", initiators=(_sdc(setup, 0), _sdc(setup, 1))) + with _client(setup) as c, pytest.raises(ArrayError, match="at most 1"): + c.ensure_host(spec) + + +def test_admin_named_sdc_and_ignore_mode(setup): + sid = setup.fake.add_sdc("ABCDEF01-0000-0000-0000-000000000001", name="esx-prod-7") + spec = HostSpec(name="worker-1", initiators=( + Initiator(protocol=Protocol.SDC, address="ABCDEF01-0000-0000-0000-000000000001"),)) + with _client(setup) as c, pytest.raises(ArrayConflictError): + c.ensure_host(spec) + with _client(setup, sdc_naming="ignore") as c: + assert c.ensure_host(spec) == sid + assert setup.fake.sdcs[sid]["name"] == "esx-prod-7" + + +def test_mapping_multi_sdc_and_null_mapped_info(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), 8 * GIB) + assert setup.fake.volumes[vol.id]["mappedSdcInfo"] is None + assert c.list_mappings(vol.id) == [] + a = c.ensure_host(HostSpec(name="w-a", initiators=(_sdc(setup, 0),))) + b = c.ensure_host(HostSpec(name="w-b", initiators=(_sdc(setup, 1),))) + m = c.map_volume(vol.id, a) + assert m.lun is None + c.map_volume(vol.id, b) # needs allowMultipleMappings + assert {x.host_id for x in c.list_mappings(vol.id)} == {a, b} + c.unmap_volume(vol.id, a) + c.unmap_volume(vol.id, b) + assert setup.fake.volumes[vol.id]["mappedSdcInfo"] is None + assert c.delete_volume(vol.id) + + +def test_remove_mapped_volume_refused_by_array(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), 8 * GIB) + host = c.ensure_host(HostSpec(name="w-a", initiators=(_sdc(setup, 0),))) + c.map_volume(vol.id, host) + adapter = c._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="mapped"): + adapter.delete_volume(vol.id) + + +def test_session_expiry_relogs_in(setup): + with _client(setup) as c: + c.get_info() + setup.fake.expire_sessions() + assert c.find_volume(str(uuid.uuid4())) is None diff --git a/tests/unit/storage_arrays/test_dell_powermax.py b/tests/unit/storage_arrays/test_dell_powermax.py new file mode 100644 index 0000000000..b3eceedfba --- /dev/null +++ b/tests/unit/storage_arrays/test_dell_powermax.py @@ -0,0 +1,256 @@ +"""Dell PowerMax specifics beyond the shared contract: cylinder sizing, +async jobs, one storage group + masking view per volume/host, hex LUNs, +non-unique volume identifiers.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters.dell_powermax import ( + CYLINDER_BYTES, + DellPowerMaxAdapter, +) +from simplyblock_core.storage_arrays.base import ( + AdapterContext, + ArrayConflictError, + ArrayError, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.http import RestSession +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.dell_powermax import ( + SYMMETRIX_ID, + contract_setup, +) + +GIB = 1 << 30 + + +class _Provider: + def __init__(self, credential): + self.credential = credential + + def resolve(self, ref, context): + return self.credential + + def invalidate(self, ref=None): + pass + + +def _client(setup) -> ArrayClient: + return ArrayClient(setup.config, _Provider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + + +def _adapter(setup, **options) -> DellPowerMaxAdapter: + opts = dict(setup.config.options) + opts.update(options) + opts = {k: v for k, v in opts.items() if v is not None} + config = setup.config.model_copy(update={"options": opts}) + session = RestSession.for_config(config, transport=FakeTransport(setup.fake), + sleep=lambda s: None) + adapter = DellPowerMaxAdapter(AdapterContext(config, setup.credential, session)) + adapter.login() + return adapter + + +def _iscsi(name: str) -> HostSpec: + return HostSpec(name=name, initiators=( + Initiator(protocol=Protocol.ISCSI, address=f"iqn.2026-01.io.sb:{name}"),)) + + +@pytest.fixture() +def setup(): + return contract_setup() + + +def test_cylinder_granularity(setup): + client = _client(setup) + vol = client.create_volume(str(uuid.uuid4()), 10 * GIB) + assert CYLINDER_BYTES == 1_966_080 + assert vol.size_bytes % CYLINDER_BYTES == 0 + assert vol.size_bytes == 5462 * CYLINDER_BYTES # ceil(10 GiB / 1920 KiB) + assert setup.fake.volumes[vol.id]["cap_cyl"] == 5462 + + +def test_one_storage_group_per_volume(setup): + client = _client(setup) + vol = client.create_volume(str(uuid.uuid4()), GIB) + sg = setup.fake.storage_groups[vol.name] + assert sg["volumes"] == [vol.id] and sg["srp"] == "SRP_1" and sg["slo"] == "NONE" + assert vol.wwn == "naa." + setup.fake.volumes[vol.id]["effective_wwn"].lower() + assert client.delete_volume(vol.id) + assert vol.name not in setup.fake.storage_groups + assert setup.fake.volumes == {} + + +def test_service_level_is_applied(setup): + adapter = _adapter(setup, service_level="Diamond") + vol = adapter.create_volume("sb-diamond", GIB) + assert setup.fake.storage_groups[vol.name]["slo"] == "Diamond" + + +def test_async_jobs_are_polled(setup): + adapter = _adapter(setup, async_jobs=True) + vol = adapter.create_volume("sb-async", 2 * GIB) + vol = adapter.resize_volume(vol.id, 4 * GIB) + assert vol.size_bytes >= 4 * GIB + host = adapter.ensure_host(_iscsi("sbtest-async")) + m = adapter.map_volume(vol.id, host) + assert m.lun == 1 + adapter.unmap_volume(vol.id, host) + adapter.delete_volume(vol.id) + jobs = setup.fake.jobs.values() + assert len(jobs) >= 5 and all(j["status"] == "SUCCEEDED" for j in jobs) + assert all(j["_polls"] == 2 for j in jobs) + assert any("/system/job/" in path for _, path in setup.fake.requests) + + +def test_failed_job_is_classified(setup): + adapter = _adapter(setup, async_jobs=True) + vol = adapter.create_volume("sb-jobfail", 4 * GIB) + with pytest.raises(ArrayRequestError, match="FAILED.*larger"): + adapter.resize_volume(vol.id, 2 * GIB) + adapter.ensure_host(_iscsi("sbtest-j1")) + with pytest.raises(ArrayConflictError, match="already in use"): + adapter.ensure_host(HostSpec(name="sbtest-j2", + initiators=_iscsi("sbtest-j1").initiators)) + + +def test_job_timeout(setup): + adapter = _adapter(setup, async_jobs=True, job_timeout_s=2) + with pytest.raises(ArrayRequestError, match="did not finish"): + adapter.create_volume("sb-slow", GIB) + + +def test_hex_lun_addresses(setup): + setup.fake.first_lun = 0x1A + adapter = _adapter(setup) + vol = adapter.create_volume("sb-hexlun", GIB) + host = adapter.ensure_host(_iscsi("sbtest-hex")) + assert adapter.map_volume(vol.id, host).lun == 26 + assert adapter.list_mappings(vol.id)[0].lun == 26 + + +def test_port_group_by_protocol(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-pg", GIB) + fc = adapter.ensure_host(HostSpec(name="sbtest-fc", initiators=( + Initiator(protocol=Protocol.FC, address="21:00:00:24:FF:4C:1A:2B"),))) + assert setup.fake.hosts[fc]["initiator"] == ["21000024ff4c1a2b"] + adapter.map_volume(vol.id, fc) + view = setup.fake.views[DellPowerMaxAdapter.view_name(vol.name, fc)] + assert view["pg"] == "sb_pg_fc" + + +def test_single_port_group_name(setup): + adapter = _adapter(setup, port_group="sb_pg_iscsi") + vol = adapter.create_volume("sb-pg1", GIB) + host = adapter.ensure_host(_iscsi("sbtest-pg1")) + adapter.map_volume(vol.id, host) + assert setup.fake.views[f"{vol.name}_{host}"]["pg"] == "sb_pg_iscsi" + + +def test_map_without_port_group_is_refused(setup): + adapter = _adapter(setup, port_group=None) + vol = adapter.create_volume("sb-nopg", GIB) + host = adapter.ensure_host(_iscsi("sbtest-nopg")) + with pytest.raises(ArrayError, match="port_group"): + adapter.map_volume(vol.id, host) + with pytest.raises(ArrayNotFoundError): + adapter.map_volume(vol.id, "sbtest-missing") + + +def test_long_host_name_gets_hashed_view_name(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-" + "a" * 24, GIB) + name = "sbtest-" + "h" * 57 + host = adapter.ensure_host(_iscsi(name)) + adapter.map_volume(vol.id, host) + (view,) = setup.fake.views + assert len(view) <= 64 and view.startswith(vol.name + "_") + assert adapter.map_volume(vol.id, host).host_id == host # idempotent + + +def test_duplicate_identifiers(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-dup", GIB) + setup.fake.add_volume("sb-dup", GIB) # same identifier, outside our SG + assert adapter.find_volume("sb-dup") == vol + setup.fake.add_volume("sb-dup2", GIB) + setup.fake.add_volume("sb-dup2", GIB) + with pytest.raises(ArrayConflictError, match="2 PowerMax devices"): + adapter.find_volume("sb-dup2") + + +def test_leftover_empty_storage_group_is_reused(setup): + adapter = _adapter(setup) + setup.fake.storage_groups["sb-left"] = setup.fake._sg_record("sb-left", "SRP_1", None) + vol = adapter.create_volume("sb-left", GIB) + assert setup.fake.storage_groups["sb-left"]["volumes"] == [vol.id] + with pytest.raises(ArrayConflictError, match="already exists"): + adapter.create_volume("sb-left", GIB) + + +def test_shared_storage_group_mapping_is_not_unmapped(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-shared", GIB) + host = adapter.ensure_host(_iscsi("sbtest-shared")) + fake = setup.fake + fake.storage_groups["customer_sg"] = fake._sg_record("customer_sg", "SRP_1", None) + fake.storage_groups["customer_sg"]["volumes"].append(vol.id) + fake.volumes[vol.id]["storageGroupId"].append("customer_sg") + adapter._call("POST", f"{adapter._slo}/maskingview", json={ + "maskingViewId": "customer_mv", + "hostOrHostGroupSelection": {"useExistingHostParam": {"hostId": host}}, + "portGroupSelection": {"useExistingPortGroupParam": {"portGroupId": "sb_pg_iscsi"}}, + "storageGroupSelection": {"useExistingStorageGroupParam": { + "storageGroupId": "customer_sg"}}}) + assert [m.host_id for m in adapter.list_mappings(vol.id)] == [host] + with pytest.raises(ArrayConflictError, match="shared storage group"): + adapter.unmap_volume(vol.id, host) + + +def test_errors_and_lookups(setup): + adapter = _adapter(setup) + assert adapter.get_volume("0FFFF") is None + assert adapter.list_mappings("0FFFF") == [] + adapter.unmap_volume("0FFFF", "nobody") + adapter.delete_host("nobody") + with pytest.raises(ArrayNotFoundError): + adapter.delete_volume("0FFFF") + + +def test_get_info_and_version_check(setup): + info = _adapter(setup).get_info() + assert info.model == "PowerMax_8500" and info.serial == SYMMETRIX_ID + assert "6079" in info.version + with pytest.raises(ArrayUnsupportedError, match="92"): + _adapter(setup, api_version="92").get_info() + + +def test_unknown_symmetrix(setup): + with pytest.raises(ArrayNotFoundError): + _adapter(setup, symmetrix_id="000197900999").get_info() + + +def test_symmetrix_id_is_required(setup): + config = setup.config.model_copy(update={"options": {}}) + session = RestSession.for_config(config, transport=FakeTransport(setup.fake)) + with pytest.raises(ArrayError, match="symmetrix_id"): + DellPowerMaxAdapter(AdapterContext(config, setup.credential, session)) + + +def test_basic_auth_on_every_request(setup): + adapter = _adapter(setup) + assert adapter.session.http.auth == ("smc", "pmax-pass") + adapter.close() + assert adapter.session.http.auth is None diff --git a/tests/unit/storage_arrays/test_dell_powerstore.py b/tests/unit/storage_arrays/test_dell_powerstore.py new file mode 100644 index 0000000000..eac8016a3b --- /dev/null +++ b/tests/unit/storage_arrays/test_dell_powerstore.py @@ -0,0 +1,212 @@ +"""Dell PowerStore specifics beyond the shared contract: CSRF token and +session cookie, PostgREST paging, 8 KiB granularity, 422 classification.""" +from __future__ import annotations + +import uuid + +import pytest +from pydantic import SecretStr + +from simplyblock_core.credentials import Credential +from simplyblock_core.storage_arrays.adapters.dell_powerstore import ( + TOKEN_HEADER, + DellPowerStoreAdapter, + powerstore_nguid, + powerstore_wwn, +) +from simplyblock_core.storage_arrays.base import ( + AdapterContext, + ArrayConflictError, + ArrayError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.http import RestSession +from simplyblock_core.storage_arrays.testing import FakeTransport, make_config +from simplyblock_core.storage_arrays.testing.dell_powerstore import contract_setup + +GIB = 1 << 30 + + +class _Provider: + def __init__(self, credential): + self.credential = credential + + def resolve(self, ref, context): + return self.credential + + def invalidate(self, ref=None): + pass + + +@pytest.fixture() +def setup(): + return contract_setup() + + +@pytest.fixture() +def client(setup): + c = ArrayClient(setup.config, _Provider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + yield c + c.close() + + +def _adapter(setup, **options) -> DellPowerStoreAdapter: + config = setup.config.model_copy(update={"options": options}) + session = RestSession.for_config(config, transport=FakeTransport(setup.fake)) + adapter = DellPowerStoreAdapter(AdapterContext(config, setup.credential, session)) + adapter.login() + return adapter + + +def _iscsi(name: str) -> HostSpec: + return HostSpec(name=name, initiators=( + Initiator(protocol=Protocol.ISCSI, address=f"iqn.2026-01.io.sb:{name}"),)) + + +def test_login_keeps_cookie_and_csrf_token_not_basic_auth(setup): + adapter = _adapter(setup) + assert adapter.session.http.auth is None + assert adapter.session.headers[TOKEN_HEADER] in setup.fake.sessions.values() + assert "auth_cookie" in adapter.session.http.cookies + vol = adapter.create_volume("sb-csrf", GIB) + assert vol.size_bytes == GIB + + +def test_mutation_without_csrf_token_is_forbidden(setup): + adapter = _adapter(setup) + adapter.session.headers.pop(TOKEN_HEADER) + assert adapter.find_volume("sb-none") is None # GET needs only the cookie + with pytest.raises(ArrayError, match="403"): + adapter.create_volume("sb-nocsrf", GIB) + + +def test_close_logs_out(setup): + adapter = _adapter(setup) + assert len(setup.fake.sessions) == 1 + adapter.close() + assert setup.fake.sessions == {} + assert ("POST", "/api/rest/logout") in setup.fake.requests + + +def test_size_is_rounded_to_8k(client, setup): + vol = client.create_volume(str(uuid.uuid4()), GIB + 1) + assert vol.size_bytes == GIB + 8192 + assert setup.fake.volumes[vol.id]["size"] % 8192 == 0 + + +def test_fake_rejects_unaligned_size(setup): + adapter = _adapter(setup) + with pytest.raises(ArrayError, match="multiple of 8192"): + adapter.create_volume("sb-odd", GIB + 512) + + +def test_identity_format(client): + vol = client.create_volume(str(uuid.uuid4()), GIB) + assert vol.wwn and vol.wwn.startswith("naa.68ccf098") and vol.wwn == vol.wwn.lower() + assert vol.nguid and vol.nguid.startswith("eui.") and len(vol.nguid) == 4 + 32 + assert powerstore_wwn("NAA.68CCF098ABC") == "naa.68ccf098abc" + assert powerstore_nguid("nguid.ABCD") == "eui.abcd" + assert powerstore_wwn(None) is None and powerstore_nguid("") is None + + +def test_select_and_filter_are_sent(client, setup): + client.find_volume(str(uuid.uuid4())) + adapter = client._adapter + assert adapter is not None + # unknown select fields are refused by the array (400), not silently ignored + with pytest.raises(ArrayError, match="400"): + adapter._list("/volume", "id,bogus") + + +def test_paged_host_lookup(setup): + setup.fake.page_limit = 2 + adapter = _adapter(setup) + ids = [adapter.ensure_host(_iscsi(f"sbtest-page-{i}")) for i in range(5)] + assert len(set(ids)) == 5 + # the last host sits on the third page of an unfiltered listing + assert len(adapter._list("/host", "id,name")) == 5 + assert adapter.ensure_host(_iscsi("sbtest-page-4")) == ids[4] + + +def test_fc_initiators_are_colon_separated_and_deduplicated(setup): + adapter = _adapter(setup) + spec = HostSpec(name="sbtest-fc", initiators=( + Initiator(protocol=Protocol.FC, address="0x21000024FF4C1A2B"),)) + host_id = adapter.ensure_host(spec) + ports = setup.fake.hosts[host_id]["host_initiators"] + assert [(p["port_name"], p["port_type"]) for p in ports] == [ + ("21:00:00:24:ff:4c:1a:2b", "FC")] + grown = HostSpec(name="sbtest-fc", initiators=spec.initiators + ( + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c"),)) + assert adapter.ensure_host(grown) == host_id + assert adapter.ensure_host(grown) == host_id + assert len(setup.fake.hosts[host_id]["host_initiators"]) == 2 + + +def test_nvme_host_uses_nvme_port_type(setup): + adapter = _adapter(setup) + spec = HostSpec(name="sbtest-nvme", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:x"),)) + host_id = adapter.ensure_host(spec) + assert setup.fake.hosts[host_id]["host_initiators"][0]["port_type"] == "NVMe" + assert setup.fake.hosts[host_id]["os_type"] == "Linux" + + +def test_initiator_conflict_is_classified(setup): + adapter = _adapter(setup) + adapter.ensure_host(_iscsi("sbtest-one")) + other = HostSpec(name="sbtest-two", initiators=_iscsi("sbtest-one").initiators) + with pytest.raises(ArrayConflictError, match="already registered"): + adapter.ensure_host(other) + + +def test_delete_attached_volume_is_conflict(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-attached", GIB) + host = adapter.ensure_host(_iscsi("sbtest-att")) + m = adapter.map_volume(vol.id, host) + assert m.lun == 1 + with pytest.raises(ArrayConflictError): + adapter.delete_volume(vol.id) + with pytest.raises(ArrayConflictError): + adapter.delete_host(host) + adapter.unmap_volume(vol.id, host) + adapter.delete_volume(vol.id) + assert adapter.get_volume(vol.id) is None + + +def test_luns_are_per_host(setup): + adapter = _adapter(setup) + host = adapter.ensure_host(_iscsi("sbtest-luns")) + luns = [adapter.map_volume(adapter.create_volume(f"sb-lun{i}", GIB).id, host).lun + for i in range(3)] + assert luns == [1, 2, 3] + + +def test_appliance_option_and_unknown_option(setup): + adapter = _adapter(setup, appliance_id="A2") + vol = adapter.create_volume("sb-a2", GIB) + assert setup.fake.volumes[vol.id]["appliance_id"] == "A2" + with pytest.raises(ArrayError, match="Unknown"): + _adapter(setup, pool="x") + + +def test_get_info(client): + info = client.get_info() + assert info.model == "PowerStore 1200T" + assert info.version == "PowerStoreOS 4.0.0.0" + assert info.serial == "PS4a7b5c3e9f01" + + +def test_api_token_credential_is_refused(): + setup = contract_setup() + config = make_config("dell-powerstore", "https://ps.example") + session = RestSession.for_config(config, transport=FakeTransport(setup.fake)) + with pytest.raises(ArrayError, match="username"): + DellPowerStoreAdapter(AdapterContext( + config, Credential({"api_token": SecretStr("t")}), session)) diff --git a/tests/unit/storage_arrays/test_framework.py b/tests/unit/storage_arrays/test_framework.py new file mode 100644 index 0000000000..5468600432 --- /dev/null +++ b/tests/unit/storage_arrays/test_framework.py @@ -0,0 +1,223 @@ +"""Vendor-neutral pieces: registry, naming/sizing, config validation, +RestSession error mapping and retries, HTTPS mock server end to end.""" +from __future__ import annotations + +import threading +import uuid + +import pytest +import requests +from pydantic import SecretStr, ValidationError + +from simplyblock_core.credentials import Credential +from simplyblock_core.storage_arrays.base import ( + ArrayAuthError, + ArrayConfig, + ArrayConflictError, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + HostSpec, + Initiator, + Protocol, + colon_wwpn, + normalize_wwpn, +) +from simplyblock_core.storage_arrays.client import ( + ArrayClient, + credential_context, + managed_name, + round_size, +) +from simplyblock_core.storage_arrays.http import RestSession +from simplyblock_core.storage_arrays.registry import ADAPTERS, ALIASES, canonical_vendor +from simplyblock_core.storage_arrays.testing import ( + FakeArray, + FakeTransport, + Reply, + route, + serve, +) +from simplyblock_core.storage_arrays.testing.pure import FakePureFlashArray + +from .conftest import StaticProvider + + +def _config(**kw): + base = dict(id="a1", vendor="pure", endpoint="https://pure01.example.com", + credential_ref={"kind": "vault-kv", "mount": "m", "path": "p"}) + base.update(kw) + return ArrayConfig(**base) + + +class TestRegistry: + + def test_all_eleven_systems_registered(self): + assert len(ADAPTERS) == 11 + + def test_aliases_resolve(self): + for alias, target in ALIASES.items(): + assert canonical_vendor(alias) == target + assert canonical_vendor(" Pure ") == "pure-flasharray" + + def test_unknown_vendor(self): + with pytest.raises(ArrayUnsupportedError, match="supported"): + canonical_vendor("acme-array") + + +class TestNamingAndSizing: + + def test_managed_name_fits_31_chars(self): + u = str(uuid.uuid4()) + assert managed_name(u, 31) == "sb-" + uuid.UUID(u).hex[:24] + assert len(managed_name(u, 20)) == 20 + + def test_managed_name_requires_uuid(self): + with pytest.raises(ValueError): + managed_name("not-a-uuid", 63) + + @pytest.mark.parametrize("size, gran, minimum, expected", [ + (1, 512, 1 << 20, 1 << 20), + ((1 << 30) + 1, 512, 1 << 20, (1 << 30) + 512), + (10 << 30, 8 << 30, 8 << 30, 16 << 30), + (8 << 30, 8 << 30, 8 << 30, 8 << 30), + ]) + def test_round_size(self, size, gran, minimum, expected): + assert round_size(size, gran, minimum) == expected + + def test_credential_context_normalizes_endpoint(self): + assert credential_context(_config(endpoint="https://PURE01.example.com/")) == \ + "array/a1/https://pure01.example.com:443" + assert credential_context(_config(endpoint="https://u.example:8443/univmax")) == \ + "array/a1/https://u.example:8443" + + +class TestConfigAndModel: + + def test_https_only(self): + with pytest.raises(ValidationError): + _config(endpoint="http://pure01.example.com") + + def test_wwpn_forms(self): + assert normalize_wwpn("21:00:00:24:FF:4C:1A:2B") == "21000024ff4c1a2b" + assert normalize_wwpn("0x21000024ff4c1a2b") == "21000024ff4c1a2b" + assert colon_wwpn("21000024FF4C1A2B") == "21:00:00:24:ff:4c:1a:2b" + with pytest.raises(ValueError): + normalize_wwpn("2100") + + def test_host_cannot_mix_scsi_and_nvme(self): + with pytest.raises(ValidationError): + HostSpec(name="h", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.x"), + Initiator(protocol=Protocol.NVME_TCP, address="nqn.x"))) + HostSpec(name="h", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.x"), + Initiator(protocol=Protocol.FC, address="2100000000000001"))) + + def test_unknown_option_is_refused(self): + provider = StaticProvider(Credential({"api_token": SecretStr("pure-token")})) + client = ArrayClient(_config(options={"bogus": 1}), provider, # type: ignore[arg-type] + transport=FakeTransport(FakePureFlashArray())) + with pytest.raises(Exception, match="Unknown .* options: bogus"): + client.get_info() + + def test_wrong_credential_shape_is_refused(self): + provider = StaticProvider(Credential({"username": SecretStr("u"), + "password": SecretStr("p")})) + client = ArrayClient(_config(), provider, # type: ignore[arg-type] + transport=FakeTransport(FakePureFlashArray())) + with pytest.raises(ArrayAuthError, match="needs credential fields api_token"): + client.get_info() + + +class _Flaky(FakeArray): + def __init__(self): + super().__init__() + self.calls = 0 + + @route("GET", "/flaky") + def flaky(self, req): + self.calls += 1 + return Reply(503 if self.calls < 3 else 200, {"ok": self.calls}) + + @route("POST", "/flaky") + def flaky_post(self, req): + self.calls += 1 + return Reply(503, {}) + + @route("GET", "/status/(?P\\d+)") + def status(self, req, code): + return Reply(int(code), {"error": "x"}) + + +class TestRestSession: + + def _session(self, fake): + return RestSession("https://x.example", transport=FakeTransport(fake), + sleep=lambda s: None) + + def test_idempotent_requests_are_retried(self): + fake = _Flaky() + assert self._session(fake).get("/flaky").json() == {"ok": 3} + + def test_non_idempotent_requests_are_not_retried(self): + fake = _Flaky() + with pytest.raises(ArrayRequestError): + self._session(fake).post("/flaky") + assert fake.calls == 1 + + @pytest.mark.parametrize("code, exc", [ + (401, ArrayAuthError), (403, ArrayAuthError), (404, ArrayNotFoundError), + (409, ArrayConflictError), (400, ArrayRequestError), (500, ArrayRequestError), + ]) + def test_status_mapping(self, code, exc): + with pytest.raises(exc): + self._session(_Flaky()).get(f"/status/{code}") + + def test_ok_statuses_are_returned(self): + resp = self._session(_Flaky()).get("/status/400", ok=(400,)) + assert resp.status_code == 400 + + def test_ca_bundle_file_is_removed_on_close(self): + import os + session = RestSession.for_config(_config(ca_bundle="-----BEGIN CERTIFICATE-----\n")) + path = session.http.verify + assert isinstance(path, str) and os.path.exists(path) + session.close() + assert not os.path.exists(path) + + def test_verify_false_only_when_configured(self): + assert RestSession.for_config(_config()).http.verify is True + assert RestSession.for_config(_config(verify_tls=False)).http.verify is False + + +class TestHttpsMockServer: + + def test_end_to_end_over_tls_with_pinned_ca(self): + fake = FakePureFlashArray(api_token="t0k") + server = serve(fake, "127.0.0.1", 0) + port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + with open(server.certfile) as f: + ca = f.read() + provider = StaticProvider(Credential({"api_token": SecretStr("t0k")})) + config = _config(endpoint=f"https://127.0.0.1:{port}", ca_bundle=ca) + with ArrayClient(config, provider) as client: # type: ignore[arg-type] + vol = client.create_volume(str(uuid.uuid4()), 1 << 30) + assert client.get_volume(vol.id) == vol + assert client.delete_volume(vol.id) + + # Without the pinned CA the self-signed server is refused. + with ArrayClient(_config(endpoint=f"https://127.0.0.1:{port}"), + provider) as client: # type: ignore[arg-type] + with pytest.raises(ArrayRequestError, match="SSLError"): + client.get_info() + finally: + server.shutdown() + server.server_close() + + def test_requests_session_is_plain(self): + # FakeTransport is mounted per session, never globally. + assert not isinstance(requests.Session().get_adapter("https://x"), FakeTransport) diff --git a/tests/unit/storage_arrays/test_hitachi_vsp.py b/tests/unit/storage_arrays/test_hitachi_vsp.py new file mode 100644 index 0000000000..f020fbeb60 --- /dev/null +++ b/tests/unit/storage_arrays/test_hitachi_vsp.py @@ -0,0 +1,360 @@ +"""Hitachi VSP specifics beyond the shared contract: path layouts, async +jobs, LDEV labels and paging, per-port host groups, LUN paths, locks.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters import hitachi_vsp as vsp +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.hitachi_vsp import contract_setup + +from .conftest import StaticProvider + +GIB = 1 << 30 +FC_A = Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b") +FC_B = Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c") +IQN_A = Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:host-a") + + +def _client(**options): + setup = contract_setup(**options) + sleeps: list[float] = [] + client = ArrayClient(setup.config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": sleeps.append}) + return client, setup.fake, sleeps + + +@pytest.fixture() +def env(): + client, fake, sleeps = _client() + yield client, fake, sleeps + client.close() + + +def _uuid() -> str: + return str(uuid.uuid4()) + + +def test_embedded_paths_session_header_and_logout(env): + client, fake, _ = env + client.get_info() + assert ("POST", "/ConfigurationManager/v1/objects/sessions") in fake.requests + assert ("GET", "/ConfigurationManager/v1/objects/storages/instance") in fake.requests + assert len(fake.sessions) == 1 + client.close() + assert fake.sessions == {} + assert fake.requests[-1][0] == "DELETE" + assert fake.requests[-1][1].startswith("/ConfigurationManager/v1/objects/sessions/") + + +def test_info_fields(env): + client, fake, _ = env + info = client.get_info() + assert info.model == "VSP E990" + assert info.serial == str(fake.serial) + assert info.version.startswith("93-") + + +def test_server_mode_uses_storage_device_prefix(): + setup_fake = contract_setup().fake + client, fake, _ = _client(storage_device_id=setup_fake.storage_device_id) + try: + prefix = f"/ConfigurationManager/v1/objects/storages/{fake.storage_device_id}" + info = client.get_info() + assert info.serial == str(fake.serial) + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-srv", initiators=(FC_A,))) + client.map_volume(vol.id, host) + assert client.delete_volume(vol.id) + paths = [p for _, p in fake.requests] + assert all(p.startswith(prefix) for p in paths) + assert any(p.startswith(f"{prefix}/jobs/") for p in paths) + assert not any(p.startswith("/ConfigurationManager/v1/objects/ldevs") for p in paths) + finally: + client.close() + + +def test_wrong_storage_device_id_is_not_found(): + client, _, _ = _client(storage_device_id="938000999999") + try: + with pytest.raises(ArrayError): + client.get_info() + finally: + client.close() + + +def test_create_waits_for_jobs_and_labels_ldev(env): + client, fake, sleeps = env + fake.job_polls = 3 + vol = client.create_volume(_uuid(), 10 * GIB + 1) + lid = int(vol.id) + assert fake.ldevs[lid]["label"] == vol.name + assert vol.size_bytes == 10 * GIB + (1 << 20) # 1 MiB granularity + assert fake.ldevs[lid]["blocks"] * 512 == vol.size_bytes + assert vol.wwn == "naa." + fake.naa(lid).lower() + assert sleeps, "job polling must go through the session's sleep" + methods = [m for m, p in fake.requests if "/jobs/" in p] + assert len(methods) >= 2 * 4 # 2 jobs x (3 running + 1 completed) + + +def test_job_timeout_is_reported_without_real_sleep(): + client, fake, sleeps = _client(job_timeout_s=10) + try: + client.get_info() + fake.job_polls = 10**6 + with pytest.raises(ArrayRequestError, match="not completed after 10s"): + client.create_volume(_uuid(), GIB) + assert 10 <= sum(sleeps) < 20 + finally: + client.close() + + +def test_failed_job_maps_storage_error(env): + client, fake, _ = env + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(FC_A,))) + client.map_volume(vol.id, host) + # LDEV delete with an LU path left fails inside the job (KART30000-E + SSB) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="KART30000-E.*LU paths"): + adapter.delete_volume(vol.id) + assert int(vol.id) in fake.ldevs + + +def test_deleted_ldev_reads_not_defined(env): + client, fake, _ = env + vol = client.create_volume(_uuid(), GIB) + client.delete_volume(vol.id) + assert int(vol.id) not in fake.ldevs + assert client.get_volume(vol.id) is None + assert client.get_volume("not-a-number") is None + + +def test_label_failure_deletes_the_new_ldev(env): + client, fake, _ = env + client.get_info() + fake.fail_next(500, {"errorSource": "x", "message": "label failed", + "messageId": "KART30000-E"}, path=r"/ldevs/\d+$") + with pytest.raises(ArrayRequestError, match="label failed"): + client.create_volume(_uuid(), GIB) + assert fake.ldevs == {} + + +def test_find_pages_through_ldevs(env, monkeypatch): + client, fake, _ = env + monkeypatch.setattr(vsp, "PAGE_SIZE", 2) + for i in range(5): + fake.add_ldev(f"other-{i}", GIB) + vol_uuid = _uuid() + vol = client.create_volume(vol_uuid, GIB) + assert int(vol.id) == 5 + assert client.find_volume(vol_uuid) == vol + heads = [p for m, p in fake.requests if m == "GET" and p.endswith("/ldevs")] + assert len(heads) >= 3 + + +def test_duplicate_labels_conflict(env): + client, fake, _ = env + vol_uuid = _uuid() + name = client.volume_name(vol_uuid) + fake.add_ldev(name, GIB) + fake.add_ldev(name, GIB) + with pytest.raises(ArrayConflictError, match="Several LDEVs"): + client.find_volume(vol_uuid) + + +def test_ldev_range_option(): + client, fake, _ = _client(ldev_range="100-101") + try: + fake.add_ldev("outside", GIB, ldev_id=5) + a = client.create_volume(_uuid(), GIB) + b = client.create_volume(_uuid(), GIB) + assert {a.id, b.id} == {"100", "101"} + with pytest.raises(ArrayError, match="No free LDEV"): + client.create_volume(_uuid(), GIB) + finally: + client.close() + + +@pytest.mark.parametrize("options, match", [ + ({"ports": []}, "ports"), + ({"ports": ["eth0"]}, "ports"), + ({"ldev_range": "9-3"}, "ldev_range"), + ({"pool_id": "x"}, "pool_id"), + ({"bogus": 1}, "Unknown"), +]) +def test_bad_options(options, match): + client, _, _ = _client(**options) + with pytest.raises(ArrayError, match=match): + client.get_info() + + +def test_host_is_a_group_per_port_of_its_protocol(env): + client, fake, _ = env + host = client.ensure_host(HostSpec(name="sbtest-fc", initiators=(FC_A,))) + groups = {k: g for k, g in fake.groups.items() if g["name"] == host} + assert sorted(p for p, _ in groups) == ["CL1-A", "CL2-A"] + assert all(g["wwns"] == ["21000024ff4c1a2b"] for g in groups.values()) + assert all(g["mode"] == "LINUX/IRIX" for g in groups.values()) + + iscsi = client.ensure_host(HostSpec(name="sbtest-is", initiators=(IQN_A,))) + assert sorted(p for (p, _), g in fake.groups.items() if g["name"] == iscsi) == \ + ["CL1-B", "CL2-B"] + + client.delete_host(host) + assert not any(g["name"] == host for g in fake.groups.values()) + assert ("CL1-A", 0) in fake.groups # default group untouched + + +def test_host_mode_options(): + client, fake, _ = _client(host_mode="VMWARE_EX", host_mode_options=[54, 63]) + try: + client.ensure_host(HostSpec(name="sbtest-vm", initiators=(FC_A,))) + g = next(g for g in fake.groups.values() if g["name"] == "sbtest-vm") + assert g["mode"] == "VMWARE_EX" and g["modeOptions"] == [54, 63] + finally: + client.close() + + +def test_no_port_for_protocol(): + client, _, _ = _client(ports=["CL1-B"]) + try: + with pytest.raises(ArrayError, match="no configured fc port"): + client.ensure_host(HostSpec(name="sbtest-fc", initiators=(FC_A,))) + finally: + client.close() + + +def test_iscsi_ownership_is_enforced_by_scan(env): + """The array lets one IQN into several targets of a port; the adapter must not.""" + client, fake, _ = env + client.ensure_host(HostSpec(name="sbtest-a", initiators=(IQN_A,))) + with pytest.raises(ArrayConflictError, match="sbtest-a"): + client.ensure_host(HostSpec(name="sbtest-b", initiators=(IQN_A,))) + assert not any(g["name"] == "sbtest-b" for g in fake.groups.values()) + + +def test_fc_wwn_in_unmanaged_group_conflicts(env): + client, fake, _ = env + client.get_info() + fake.groups[("CL2-A", 0)]["wwns"].append("21000024ff4c1a2c") + with pytest.raises(ArrayConflictError, match="CL2-A"): + client.ensure_host(HostSpec(name="sbtest-b", initiators=(FC_B,))) + + +def test_same_lun_on_every_port(env): + client, fake, _ = env + other = client.create_volume(_uuid(), GIB) + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(FC_A,))) + # occupy LUN 0 on the second port only + num = next(n for (p, n), g in fake.groups.items() if p == "CL2-A" and g["name"] == host) + fake.groups[("CL2-A", num)]["luns"][0] = int(other.id) + m = client.map_volume(vol.id, host) + assert m.lun == 1 + luns = {p: lun for (p, n), g in fake.groups.items() if g["name"] == host + for lun, lid in g["luns"].items() if lid == int(vol.id)} + assert luns == {"CL1-A": 1, "CL2-A": 1} + assert client.list_mappings(vol.id) == [m] + + +def test_map_repairs_missing_port_path(env): + client, fake, _ = env + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(FC_A,))) + m = client.map_volume(vol.id, host) + num = next(n for (p, n), g in fake.groups.items() if p == "CL2-A" and g["name"] == host) + fake.groups[("CL2-A", num)]["luns"].clear() + assert client.map_volume(vol.id, host) == m + assert fake.groups[("CL2-A", num)]["luns"] == {m.lun: int(vol.id)} + + +def test_unmap_removes_every_port_path(env): + client, fake, _ = env + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(FC_A,))) + client.map_volume(vol.id, host) + client.unmap_volume(vol.id, host) + assert not any(g["luns"] for g in fake.groups.values()) + client.unmap_volume(vol.id, host) + + +def test_map_to_unknown_host_is_not_found(env): + client, _, _ = env + vol = client.create_volume(_uuid(), GIB) + with pytest.raises(ArrayError, match="no host group"): + client.map_volume(vol.id, "sbtest-missing") + + +def test_lock_required_without_use_lock_fails(): + client, fake, _ = _client() + try: + fake.require_lock = True + with pytest.raises(ArrayError, match="not locked"): + client.create_volume(_uuid(), GIB) + finally: + client.close() + + +def test_use_lock_wraps_changes(): + client, fake, _ = _client(use_lock=True) + try: + fake.require_lock = True + vol = client.create_volume(_uuid(), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(FC_A,))) + client.map_volume(vol.id, host) + client.resize_volume(vol.id, 2 * GIB) + assert client.delete_volume(vol.id) + client.delete_host(host) + assert fake.lock_owner is None + locks = [p for _, p in fake.requests if p.endswith("/lock/invoke")] + unlocks = [p for _, p in fake.requests if p.endswith("/unlock/invoke")] + assert len(locks) == len(unlocks) >= 6 + finally: + client.close() + + +def test_lock_released_when_change_fails(): + client, fake, _ = _client(use_lock=True) + try: + client.get_info() + fake.require_lock = True + fake.pools.clear() + with pytest.raises(ArrayError): + client.create_volume(_uuid(), GIB) + assert fake.lock_owner is None + finally: + client.close() + + +def test_resize_expands_by_difference(env): + client, fake, _ = env + vol = client.create_volume(_uuid(), 2 * GIB) + grown = client.resize_volume(vol.id, 3 * GIB) + assert grown.size_bytes == 3 * GIB + assert fake.ldevs[int(vol.id)]["blocks"] == 3 * GIB // 512 + + +def test_classify_error_bodies(): + nf = vsp._classify("GET x", 400, {"message": "The LDEV is not defined.", + "messageId": "KART30000-E"}) + assert type(nf).__name__ == "ArrayNotFoundError" + cf = vsp._classify("POST x", None, {"message": "already registered", + "messageId": "KART30000-E", + "errorCode": {"SSB1": "B957", "SSB2": "4184"}}) + assert isinstance(cf, ArrayConflictError) and "SSB1=B957" in str(cf) + other = vsp._classify("POST x", 503, {"message": "busy"}) + assert isinstance(other, ArrayRequestError) and other.status == 503 diff --git a/tests/unit/storage_arrays/test_hpe_nimble.py b/tests/unit/storage_arrays/test_hpe_nimble.py new file mode 100644 index 0000000000..aadf36ba31 --- /dev/null +++ b/tests/unit/storage_arrays/test_hpe_nimble.py @@ -0,0 +1,188 @@ +"""HPE Alletra 5000/6000 / Nimble adapter specifics beyond the shared contract.""" +from __future__ import annotations + +import dataclasses +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport, make_config +from simplyblock_core.storage_arrays.testing.hpe_nimble import contract_setup + +from .conftest import StaticProvider + +GIB = 1 << 30 +MIB = 1 << 20 + + +def _client(**options): + setup = contract_setup() + if options: + setup = dataclasses.replace( + setup, config=make_config("hpe-nimble", "https://nimble.fake.example:5392", + **options)) + client = ArrayClient(setup.config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def _iscsi(name, *iqns): + return HostSpec(name=name, initiators=tuple( + Initiator(protocol=Protocol.ISCSI, address=i) for i in iqns)) + + +def _fc(name, *wwpns): + return HostSpec(name=name, initiators=tuple( + Initiator(protocol=Protocol.FC, address=w) for w in wwpns)) + + +def test_token_released_by_id_on_close(): + client, fake = _client() + client.get_info() + assert len(fake.tokens) == 1 + (tid,) = fake.tokens + client.close() + assert fake.tokens == {} + assert ("DELETE", f"/v1/tokens/{tid}") in fake.requests + + +def test_port_5392_is_kept(): + setup = contract_setup() + sent: list[str] = [] + + class Recording(FakeTransport): + def send(self, request, **kw): + sent.append(request.url) + return super().send(request, **kw) + + client = ArrayClient(setup.config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=Recording(setup.fake)) + client.get_info() + assert sent and all(u.startswith("https://nimble.fake.example:5392/v1/") for u in sent) + + +def test_get_info(): + client, _ = _client() + info = client.get_info() + assert info.model == "AF40" and info.serial == "AF-123456" + assert info.version.startswith("6.") + + +def test_sizes_in_mib_and_eui_identity(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB + 1) + assert vol.size_bytes == GIB + MIB + raw = fake.volumes[vol.id] + assert raw["size"] == 1025 + assert vol.nguid is None and vol.wwn == "eui." + raw["serial_number"] + assert vol.identifier == vol.wwn + + +def test_pool_and_performance_policy_resolved_by_name(): + client, fake = _client(pool="fast", performance_policy="Oracle OLTP") + vol = client.create_volume(str(uuid.uuid4()), GIB) + raw = fake.volumes[vol.id] + assert raw["pool_name"] == "fast" and raw["perfpolicy_name"] == "Oracle OLTP" + + +def test_unknown_pool_is_an_error(): + client, fake = _client(pool="nope") + with pytest.raises(ArrayError, match="pool 'nope' not found"): + client.create_volume(str(uuid.uuid4()), GIB) + assert fake.volumes == {} + + +def test_delete_offlines_first(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + adapter = client._adapter + assert adapter is not None + # a direct DELETE of an online volume is refused by the array + with pytest.raises(ArrayConflictError, match="SM_vol_online"): + adapter._request("DELETE", f"/v1/volumes/{vol.id}") + assert client.delete_volume(vol.id) is True + calls = [(m, p) for m, p in fake.requests if p == f"/v1/volumes/{vol.id}"] + assert ("PUT", f"/v1/volumes/{vol.id}") in calls + assert calls.index(("PUT", f"/v1/volumes/{vol.id}")) < len(calls) - 1 + + +def test_fc_group_uses_colon_wwpns(): + client, fake = _client() + hid = client.ensure_host(_fc("sbfc", "2100000E1E2F3A4B")) + assert fake.groups[hid]["access_protocol"] == "fc" + assert [i["wwpn"] for i in fake.initiators.values()] == ["21:00:00:0e:1e:2f:3a:4b"] + assert client.ensure_host(_fc("sbfc", "21:00:00:0e:1e:2f:3a:4b")) == hid + assert len(fake.initiators) == 1 + + +def test_mixed_protocol_host_refused(): + client, _ = _client() + spec = HostSpec(name="sbmix", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:m"), + Initiator(protocol=Protocol.FC, address="2100000e1e2f3a4c"))) + with pytest.raises(ArrayError, match="either iSCSI or FC"): + client.ensure_host(spec) + + +def test_existing_group_of_other_protocol_conflicts(): + client, _ = _client() + client.ensure_host(_iscsi("sbsame", "iqn.2026-01.io.sb:s")) + with pytest.raises(ArrayConflictError, match="access_protocol iscsi"): + client.ensure_host(_fc("sbsame", "2100000e1e2f3a4d")) + + +def test_initiator_owned_elsewhere_conflicts_before_creating_group(): + client, fake = _client() + client.ensure_host(_iscsi("sbown", "iqn.2026-01.io.sb:o")) + with pytest.raises(ArrayConflictError, match="sbown"): + client.ensure_host(_iscsi("sbthief", "iqn.2026-01.io.sb:o")) + assert {g["name"] for g in fake.groups.values()} == {"sbown"} + + +def test_iscsi_luns_are_zero_with_volume_scoped_targets(): + client, fake = _client() + hid = client.ensure_host(_iscsi("sbvst", "iqn.2026-01.io.sb:v")) + luns = [client.map_volume(client.create_volume(str(uuid.uuid4()), GIB).id, hid).lun + for _ in range(2)] + assert luns == [0, 0] + + +def test_fc_luns_are_distinct_per_group(): + client, fake = _client() + hid = client.ensure_host(_fc("sbfc2", "2100000e1e2f3a4e")) + luns = [client.map_volume(client.create_volume(str(uuid.uuid4()), GIB).id, hid).lun + for _ in range(3)] + assert luns == [0, 1, 2] + + +def test_acr_is_applied_to_both_and_map_is_idempotent(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + hid = client.ensure_host(_iscsi("sbacr", "iqn.2026-01.io.sb:a")) + m = client.map_volume(vol.id, hid) + assert client.map_volume(vol.id, hid) == m + (acr,) = fake.acrs.values() + assert acr["apply_to"] == "both" and acr["vol_id"] == vol.id + + +def test_malformed_volume_id_is_none(): + client, _ = _client() + assert client.get_volume("not-a-nimble-id") is None + + +def test_host_in_use_cannot_be_deleted(): + client, _ = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + hid = client.ensure_host(_iscsi("sbbusy", "iqn.2026-01.io.sb:b")) + client.map_volume(vol.id, hid) + with pytest.raises(ArrayConflictError, match="SM_igrp_has_acl"): + client.delete_host(hid) diff --git a/tests/unit/storage_arrays/test_hpe_wsapi.py b/tests/unit/storage_arrays/test_hpe_wsapi.py new file mode 100644 index 0000000000..d3fef3424e --- /dev/null +++ b/tests/unit/storage_arrays/test_hpe_wsapi.py @@ -0,0 +1,188 @@ +"""HPE WSAPI adapter specifics beyond the shared contract.""" +from __future__ import annotations + +import dataclasses +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + ArrayNotFoundError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport, make_config +from simplyblock_core.storage_arrays.testing.hpe_wsapi import contract_setup + +from .conftest import StaticProvider + +GIB = 1 << 30 +MIB = 1 << 20 + + +def _client(**options): + setup = contract_setup() + if options: + setup = dataclasses.replace( + setup, config=make_config("hpe-wsapi", "https://wsapi.fake.example", + **{"cpg": "SSD_r6", **options})) + client = ArrayClient(setup.config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def _iscsi(name, *iqns): + return HostSpec(name=name, initiators=tuple( + Initiator(protocol=Protocol.ISCSI, address=i) for i in iqns)) + + +def test_session_key_header_and_logout(): + client, fake = _client() + client.get_info() + assert len(fake.sessions) == 1 + client.close() + assert fake.sessions == set() + assert any(p.startswith("/api/v1/credentials/") for m, p in fake.requests if m == "DELETE") + + +def test_get_info_reads_system(): + client, _ = _client() + info = client.get_info() + assert info.model.startswith("HPE Alletra") and info.serial == "CZ12345678" + + +def test_cpg_option_required(): + from simplyblock_core.storage_arrays.adapters.hpe_wsapi import HpeWsapiAdapter + from simplyblock_core.storage_arrays.base import AdapterContext + from simplyblock_core.storage_arrays.http import RestSession + config = make_config("hpe-wsapi", "https://wsapi.fake.example") + ctx = AdapterContext(config, contract_setup().credential, RestSession(config.endpoint)) + with pytest.raises(ArrayError, match="cpg"): + HpeWsapiAdapter(ctx) + + +def test_invalid_provisioning_rejected(): + with pytest.raises(ArrayError, match="provisioning"): + _client(provisioning="full")[0].get_info() + + +def test_create_body_tpvv_and_reduce(): + client, fake = _client() + client.create_volume(str(uuid.uuid4()), GIB) + body = fake.created_bodies[-1] + assert body["tpvv"] is True and "reduce" not in body + assert body["cpg"] == "SSD_r6" and body["sizeMiB"] == 1024 + + client, fake = _client(provisioning="reduce") + client.create_volume(str(uuid.uuid4()), GIB) + assert fake.created_bodies[-1]["reduce"] is True and "tpvv" not in fake.created_bodies[-1] + + +def test_names_fit_31_chars_and_wwn_is_naa(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + assert len(vol.name) <= 31 and vol.id == vol.name + assert vol.wwn == "naa." + fake.volumes[vol.name]["wwn"].lower() + assert vol.wwn.startswith("naa.60002ac") + + +def test_size_rounds_to_256_mib(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB + 1) + assert vol.size_bytes == GIB + 256 * MIB + assert fake.volumes[vol.name]["sizeMiB"] == 1280 + + +def test_grow_sends_increment(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), 2 * GIB) + seen = [] + orig = fake.put_volume + + def spy(req, name): + seen.append(req.json()) + return orig(req, name) + + fake._routes = [(m, p, spy if h == orig else h) for m, p, h in fake._routes] + grown = client.resize_volume(vol.id, 5 * GIB) + assert seen == [{"action": 3, "sizeMiB": 3 * 1024}] + assert grown.size_bytes == 5 * GIB + + +def test_delete_with_vlun_is_conflict_at_adapter_level(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(_iscsi("sbh1", "iqn.2026-01.io.sb:x")) + client.map_volume(vol.id, host) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="code 34"): + adapter.delete_volume(vol.id) + # the client unmaps first + assert client.delete_volume(vol.id) is True + + +def test_vlun_location_and_dedup_of_active_paths(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + h = client.ensure_host(_iscsi("sbh2", "iqn.2026-01.io.sb:y")) + fake.vluns.append({"volumeName": "other", "hostname": h, "lun": 0}) # LUN 0 taken + fake.add_volume("other", GIB) + m = client.map_volume(vol.id, h) + assert m.lun == 1 + members = [x for x in fake._vlun_members(vol.id)] + assert len(members) == 1 + fake.paths_per_host + assert client.list_mappings(vol.id) == [m] + + +def test_missing_host_on_map_is_not_found(): + client, _ = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + with pytest.raises(ArrayNotFoundError, match="code 17"): + client.map_volume(vol.id, "nohost") + + +def test_mixed_iscsi_fc_host_uses_two_requests(): + client, fake = _client() + spec = HostSpec(name="sbmixed", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:m"), + Initiator(protocol=Protocol.FC, address="10:00:00:00:c9:aa:bb:cc"))) + assert client.ensure_host(spec) == "sbmixed" + host = fake.hosts["sbmixed"] + assert [p["name"] for p in host["iSCSIPaths"]] == ["iqn.2026-01.io.sb:m"] + assert [p["wwn"] for p in host["FCPaths"]] == ["10000000C9AABBCC"] + assert client.ensure_host(spec) == "sbmixed" + + +def test_persona_option_used(): + client, fake = _client(persona=11) + client.ensure_host(_iscsi("sbvm", "iqn.2026-01.io.sb:v")) + assert fake.hosts["sbvm"]["persona"] == 11 + + +def test_path_owned_by_other_host_is_conflict(): + client, _ = _client() + client.ensure_host(_iscsi("sba", "iqn.2026-01.io.sb:dup")) + with pytest.raises(ArrayConflictError, match="code 73"): + client.ensure_host(_iscsi("sbb", "iqn.2026-01.io.sb:dup")) + + +def test_nvme_refused(): + client, fake = _client() + spec = HostSpec(name="sbn", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:x"),)) + with pytest.raises(ArrayError, match="does not support"): + client.ensure_host(spec) + assert not fake.requests + + +def test_host_name_limit_enforced_by_client(): + client, fake = _client() + with pytest.raises(ArrayError, match="exceeds 31"): + client.ensure_host(_iscsi("h" * 32, "iqn.2026-01.io.sb:long")) + assert not fake.requests diff --git a/tests/unit/storage_arrays/test_huawei_dorado.py b/tests/unit/storage_arrays/test_huawei_dorado.py new file mode 100644 index 0000000000..64945ff14b --- /dev/null +++ b/tests/unit/storage_arrays/test_huawei_dorado.py @@ -0,0 +1,172 @@ +"""Huawei OceanStor Dorado specifics beyond the shared contract suite.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayAuthError, + ArrayConflictError, + ArrayError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.huawei_dorado import contract_setup + +GIB = 1 << 30 +IQN_A = Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:a") +IQN_B = Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:b") + + +class _Provider: + def __init__(self, credential): + self.credential = credential + self.invalidated = 0 + + def resolve(self, ref, context): + return self.credential + + def invalidate(self, ref=None): + self.invalidated += 1 + + +def _client(**options): + setup = contract_setup() + if options: + setup.config = setup.config.model_copy( + update={"options": {**setup.config.options, **options}}) + client = ArrayClient(setup.config, _Provider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def _envelope(code, description="x"): + return {"data": {}, "error": {"code": code, "description": description}} + + +def test_login_and_device_scoped_paths(): + client, fake = _client() + info = client.get_info() + assert info.serial == fake.device_id and info.version == "6.1.5" + assert info.model == "OceanStor Dorado 5000 V6" + paths = [p for _, p in fake.requests] + assert paths[0] == "/deviceManager/rest/xx/sessions" + assert paths[1] == f"/deviceManager/rest/{fake.device_id}/system/" + client.close() + assert fake.requests[-1] == ("DELETE", f"/deviceManager/rest/{fake.device_id}/sessions") + assert not fake.tokens + + +def test_capacity_is_in_sectors_and_rounded_to_mib(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), 10 * GIB + 1) + assert vol.size_bytes == 10 * GIB + (1 << 20) + assert fake.luns[vol.id]["CAPACITY"] == str((10 * GIB + (1 << 20)) // 512) + assert fake.luns[vol.id]["ALLOCTYPE"] == "1" + assert vol.wwn == "naa." + fake.luns[vol.id]["WWN"].lower() + assert len(vol.name) <= 31 + + +def test_unknown_pool(): + client, _ = _client(storage_pool="nope") + with pytest.raises(ArrayError, match="storage pool 'nope' not found"): + client.create_volume(str(uuid.uuid4()), GIB) + + +def test_http_200_error_envelopes_are_mapped(): + client, fake = _client() + client.get_info() + fake.fail_next(200, _envelope(1077948993, "The name already exists."), path="/lun$") + with pytest.raises(ArrayConflictError, match="1077948993"): + client.find_volume(str(uuid.uuid4())) + fake.fail_next(200, _envelope(1077949006, "The system is busy."), path="/lun$") + with pytest.raises(ArrayRequestError, match="1077949006"): + client.find_volume(str(uuid.uuid4())) + fake.fail_next(200, _envelope(1077936859, "The LUN does not exist."), path="/lun/") + assert client.get_volume("4242") is None + + +def test_minus_401_envelope_relogs_in_once(): + client, fake = _client() + client.get_info() + fake.expire_sessions() + client.get_info() + logins = [p for _, p in fake.requests if p.endswith("/xx/sessions")] + assert len(logins) == 2 + + +def test_wrong_password_is_auth_error(): + client, fake = _client() + fake.set_credential(username="admin", password="other") + with pytest.raises(ArrayAuthError, match="1077987870"): + client.get_info() + + +def test_mapping_triple_and_teardown(): + client, fake = _client() + v1 = client.create_volume(str(uuid.uuid4()), GIB) + v2 = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(IQN_A,))) + m1 = client.map_volume(v1.id, host) + m2 = client.map_volume(v2.id, host) + assert (m1.lun, m2.lun) == (1, 2) # HostLUNID parsed from ASSOCIATEMETADATA + assert {g["NAME"] for g in fake.lungroups.values()} == {f"sb_lg_{host}"} + assert {g["NAME"] for g in fake.hostgroups.values()} == {f"sb_hg_{host}"} + assert {v["NAME"] for v in fake.views.values()} == {f"sb_mv_{host}"} + assert client.map_volume(v1.id, host) == m1 + + with pytest.raises(ArrayConflictError, match="still has LUNs"): + client.delete_host(host) + client.delete_volume(v1.id) + client.delete_volume(v2.id) + client.delete_host(host) + assert not (fake.hosts or fake.lungroups or fake.hostgroups or fake.views) + assert fake.initiators["iscsi_initiator"]["iqn.2026-01.io.sb:a"]["ISFREE"] == "true" + # the freed initiator can be claimed by a new host + client.ensure_host(HostSpec(name="sbtest-b", initiators=(IQN_A,))) + + +def test_initiator_bound_to_foreign_host_creates_nothing(): + client, fake = _client() + owner = client.ensure_host(HostSpec(name="customer-host", initiators=(IQN_A,))) + with pytest.raises(ArrayConflictError, match="customer-host"): + client.ensure_host(HostSpec(name="sbtest-x", initiators=(IQN_B, IQN_A))) + assert [h["NAME"] for h in fake.hosts.values()] == ["customer-host"] + assert fake.initiators["iscsi_initiator"]["iqn.2026-01.io.sb:a"]["PARENTID"] == owner + + +def test_fc_initiator_ids_are_plain_lowercase_hex(): + client, fake = _client() + host = client.ensure_host(HostSpec(name="sbtest-fc", initiators=( + Initiator(protocol=Protocol.FC, address="21:00:00:24:FF:4C:1A:2B"),))) + ini = fake.initiators["fc_initiator"]["21000024ff4c1a2b"] + assert ini["PARENTID"] == host and ini["ISFREE"] == "false" + + +def test_host_in_foreign_hostgroup_conflicts_on_map(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(HostSpec(name="sbtest-a", initiators=(IQN_A,))) + gid = str(fake.next_id()) + fake.hostgroups[gid] = {"ID": gid, "NAME": "customer_hg", "TYPE": "14", + "DESCRIPTION": "", "hosts": {host}} + with pytest.raises(ArrayConflictError, match="customer_hg"): + client.map_volume(vol.id, host) + + +def test_lun_in_customer_lungroup_is_not_listed_and_blocks_delete(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + gid = str(fake.next_id()) + fake.lungroups[gid] = {"ID": gid, "NAME": "customer_lg", "TYPE": "256", + "DESCRIPTION": "", "luns": {vol.id}} + assert client.list_mappings(vol.id) == [] + with pytest.raises(ArrayRequestError): + client.delete_volume(vol.id) + assert client.get_volume(vol.id) is not None diff --git a/tests/unit/storage_arrays/test_ibm_flashsystem.py b/tests/unit/storage_arrays/test_ibm_flashsystem.py new file mode 100644 index 0000000000..be9e3d5d52 --- /dev/null +++ b/tests/unit/storage_arrays/test_ibm_flashsystem.py @@ -0,0 +1,187 @@ +"""IBM Storage Virtualize specifics beyond the shared contract suite.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.ibm_flashsystem import contract_setup + +GIB = 1 << 30 + + +class _Provider: + def __init__(self, credential): + self.credential = credential + self.invalidated = 0 + + def resolve(self, ref, context): + return self.credential + + def invalidate(self, ref=None): + self.invalidated += 1 + + +def _client(**options): + setup = contract_setup() + if options: + setup.config = setup.config.model_copy( + update={"options": {**setup.config.options, **options}}) + client = ArrayClient(setup.config, _Provider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def _cmds(fake, name): + return [opts for cmd, _, opts in fake.commands if cmd == name] + + +def test_get_info(): + client, _ = _client() + info = client.get_info() + assert info.model == "IBM FlashSystem 7300" + assert info.version == "8.6.0.4" + assert info.serial == "000002042140A0B2" + + +def test_mkvolume_parameters_thin_default(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), 10 * GIB + 1) + (opts,) = _cmds(fake, "mkvolume") + assert opts == {"name": vol.name, "pool": "Pool0", "size": 10 * GIB + 512, + "unit": "b", "thin": True} + assert vol.size_bytes == 10 * GIB + 512 + assert vol.wwn == "naa." + fake.vdisks[vol.id]["vdisk_UID"].lower() + # capacity must be read in bytes, never in the human-readable form + assert all(o.get("bytes") is True for o in _cmds(fake, "lsvdisk")) + + +def test_thick_volume_and_io_group(): + client, fake = _client(thin=False, io_group="io_grp1") + vol = client.create_volume(str(uuid.uuid4()), GIB) + (opts,) = _cmds(fake, "mkvolume") + assert "thin" not in opts and opts["iogrp"] == "io_grp1" + assert fake.vdisks[vol.id]["IO_group_name"] == "io_grp1" + + +def test_rest_v1_root(): + client, fake = _client(rest_root="/rest/v1") + client.create_volume(str(uuid.uuid4()), GIB) + assert all(path.startswith("/rest/v1/") for _, path in fake.requests) + + +def test_resize_sends_delta(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), 8 * GIB) + client.resize_volume(vol.id, 20 * GIB) + assert _cmds(fake, "expandvdisksize") == [{"size": 12 * GIB, "unit": "b"}] + + +def test_unknown_pool_is_not_found(): + client, _ = _client(pool="NoSuchPool") + with pytest.raises(ArrayNotFoundError, match="CMMVC5754E"): + client.create_volume(str(uuid.uuid4()), GIB) + + +def test_cmmvc_codes_are_mapped(): + client, fake = _client() + fake.fail_next(500, b"CMMVC6035E The action failed as the object already exists.", + path="mkvolume") + with pytest.raises(ArrayConflictError, match="CMMVC6035E"): + client.create_volume(str(uuid.uuid4()), GIB) + fake.fail_next(500, b"CMMVC5786E The action failed because the cluster is not in a " + b"stable state.", path="mkvolume") + with pytest.raises(ArrayRequestError) as e: + client.create_volume(str(uuid.uuid4()), GIB) + assert e.value.status == 500 and "CMMVC5786E" in str(e.value) + + +def test_expired_token_is_403_and_relogs_in(): + client, fake = _client() + client.get_info() + fake.expire_sessions() + client.get_info() + assert [p for _, p in fake.requests].count("/rest/auth") == 2 + + +def test_volume_id_reuse_is_caught_by_identifier(): + client, _ = _client() + old = client.create_volume(str(uuid.uuid4()), GIB) + assert client.delete_volume(old.id) + new = client.create_volume(str(uuid.uuid4()), GIB) + assert new.id == old.id # the array hands out the lowest free id again + with pytest.raises(ArrayConflictError, match="does not match"): + client.delete_volume(old.id, expected_identifier=old.identifier) + assert client.get_volume(new.id) == new + + +def test_multi_host_mapping_uses_force_and_single_row_quirk(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + a = client.ensure_host(HostSpec(name="sbtest-a", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:a"),))) + b = client.ensure_host(HostSpec(name="sbtest-b", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:b"),))) + m = client.map_volume(vol.id, a) + assert m.lun == 0 + # one mapping: lsvdiskhostmap answers with a bare dict + assert client.list_mappings(vol.id) == [m] + client.map_volume(vol.id, b) + assert all(o.get("force") is True for o in _cmds(fake, "mkvdiskhostmap")) + assert {x.host_id for x in client.list_mappings(vol.id)} == {a, b} + + +def test_fc_host_wwpns_are_colon_separated_and_forced(): + client, fake = _client() + client.ensure_host(HostSpec(name="sbtest-fc", initiators=( + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"), + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2c")))) + (opts,) = _cmds(fake, "mkhost") + assert opts == {"name": "sbtest-fc", "fcwwpn": "21000024FF4C1A2B:21000024FF4C1A2C", + "force": True} + # re-run with the same ports: nothing is re-added + client.ensure_host(HostSpec(name="sbtest-fc", initiators=( + Initiator(protocol=Protocol.FC, address="21000024ff4c1a2b"),))) + assert _cmds(fake, "addhostport") == [] + + +def test_fc_port_of_other_host_conflicts(): + client, fake = _client() + wwpn = Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b") + client.ensure_host(HostSpec(name="sbtest-one", initiators=(wwpn,))) + with pytest.raises(ArrayConflictError, match="registered to another host"): + client.ensure_host(HostSpec(name="sbtest-two", initiators=(wwpn,))) + assert "sbtest-two" not in fake.hosts + + +def test_nvme_host_protocol(): + client, fake = _client() + client.ensure_host(HostSpec(name="sbtest-nvme", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:x"),))) + assert fake.hosts["sbtest-nvme"]["protocol"] == "tcpnvme" + with pytest.raises(ArrayUnsupportedError, match="one transport"): + client.ensure_host(HostSpec(name="sbtest-mixed", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:y"), + Initiator(protocol=Protocol.NVME_RDMA, address="nqn.2014-08.org.nvmexpress:uuid:z")))) + + +def test_delete_host_with_mappings_conflicts(): + client, _ = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(HostSpec(name="sbtest-busy", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:busy"),))) + client.map_volume(vol.id, host) + with pytest.raises(ArrayConflictError, match="CMMVC5871E"): + client.delete_host(host) diff --git a/tests/unit/storage_arrays/test_infinidat.py b/tests/unit/storage_arrays/test_infinidat.py new file mode 100644 index 0000000000..6b0b3ba41f --- /dev/null +++ b/tests/unit/storage_arrays/test_infinidat.py @@ -0,0 +1,147 @@ +"""Infinidat InfiniBox specifics beyond the shared contract.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters.infinidat import infinibox_wwn +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + ArrayNotFoundError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport, make_config +from simplyblock_core.storage_arrays.testing.infinidat import contract_setup + +from .conftest import StaticProvider + +GIB = 1 << 30 + + +@pytest.fixture() +def setup(): + return contract_setup() + + +def _client(setup, **options): + config = setup.config + if options: + config = make_config("infinidat-infinibox", config.endpoint, + **{**config.options, **options}) + return ArrayClient(config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + + +def _iqn(n: str) -> Initiator: + return Initiator(protocol=Protocol.ISCSI, address=f"iqn.2026-01.io.sb:{n}") + + +def test_wwn_from_serial(): + assert infinibox_wwn("742B0F000004E2B000000000000F3A3") == "naa.6742b0f000004e2b000000000000f3a3" + assert infinibox_wwn("6742b0f000004e2b000000000000f3a3") == "naa.6742b0f000004e2b000000000000f3a3" + + +def test_volume_identity_and_pool(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), 10 * GIB + 1) + raw = setup.fake.volumes[int(vol.id)] + assert vol.wwn == "naa.6" + raw["serial"] + assert len(vol.wwn) == len("naa.") + 32 + assert raw["pool_id"] == next(iter(setup.fake.pools)) + assert raw["provtype"] == "THIN" + assert vol.size_bytes == 10 * GIB + (1 << 20) + + +def test_thick_option_and_bad_option(setup): + with _client(setup, provtype="thick") as c: + vol = c.create_volume(str(uuid.uuid4()), GIB) + assert setup.fake.volumes[int(vol.id)]["provtype"] == "THICK" + with _client(setup, provtype="sparse") as c, pytest.raises(ArrayError, match="provtype"): + c.get_info() + + +def test_missing_pool(setup): + with _client(setup, pool="nope") as c, pytest.raises(ArrayNotFoundError, match="nope"): + c.create_volume(str(uuid.uuid4()), GIB) + + +def test_destructive_requests_send_approval(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), GIB) + host = c.ensure_host(HostSpec(name="h1", initiators=(_iqn("a"),))) + c.map_volume(vol.id, host) + assert c.delete_volume(vol.id) # unmap + delete, both need approved=true + c.delete_host(host) + assert not setup.fake.volumes and not setup.fake.hosts + + +def test_approval_required_is_not_an_auth_error(setup): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), GIB) + setup.fake.fail_next(403, {"result": None, "error": { + "code": "APPROVAL_REQUIRED", "message": "needs approval"}, + "metadata": {"ready": True}}, path=r"/volumes/\d+$") + adapter = c._adapter + assert adapter is not None + with pytest.raises(ArrayRequestError, match="APPROVAL_REQUIRED"): + adapter.delete_volume(vol.id) + + +def test_conflicting_port_rolls_back_new_host(setup): + with _client(setup) as c: + c.ensure_host(HostSpec(name="h1", initiators=(_iqn("a"),))) + with pytest.raises(ArrayConflictError): + c.ensure_host(HostSpec(name="h2", initiators=(_iqn("b"), _iqn("a")))) + assert [h["name"] for h in setup.fake.hosts.values()] == ["h1"] + + +def test_fc_and_iscsi_ports_normalised(setup): + fc = Initiator(protocol=Protocol.FC, address="21:00:00:24:FF:4C:1A:2B") + with _client(setup) as c: + host = c.ensure_host(HostSpec(name="h1", initiators=(fc, _iqn("A")))) + ports = setup.fake.hosts[int(host)]["ports"] + assert {(p["type"], p["address"]) for p in ports} == { + ("FC", "21000024ff4c1a2b"), ("ISCSI", "iqn.2026-01.io.sb:a")} + # colon-formatted ports returned by the array still count as present + ports[0]["address"] = "21:00:00:24:ff:4c:1a:2b" + assert c.ensure_host(HostSpec(name="h1", initiators=(fc,))) == host + assert len(setup.fake.hosts[int(host)]["ports"]) == 2 + + +def test_luns_start_at_one_per_host(setup): + with _client(setup) as c: + host = c.ensure_host(HostSpec(name="h1", initiators=(_iqn("a"),))) + luns = [c.map_volume(c.create_volume(str(uuid.uuid4()), GIB).id, host).lun + for _ in range(3)] + assert luns == [1, 2, 3] + + +def test_pagination_walks_all_pages(setup, monkeypatch): + import simplyblock_core.storage_arrays.adapters.infinidat as mod + monkeypatch.setattr(mod, "_PAGE_SIZE", 2) + for i in range(5): + setup.fake.add_volume(f"sb-same-{i}", GIB) + with _client(setup) as c: + c.get_info() + adapter = c._adapter + assert adapter is not None + assert len(adapter._list("/volumes")) == 5 + + +def test_non_numeric_ids_are_absent(setup): + with _client(setup) as c: + assert c.get_volume("sb-abc") is None + c.delete_host("not-a-number") + + +def test_bad_credential_is_auth_error(setup): + setup.fake.set_credential(password="other") + with _client(setup) as c, pytest.raises(ArrayError): + c.get_info() diff --git a/tests/unit/storage_arrays/test_netapp_ontap.py b/tests/unit/storage_arrays/test_netapp_ontap.py new file mode 100644 index 0000000000..31d2326b76 --- /dev/null +++ b/tests/unit/storage_arrays/test_netapp_ontap.py @@ -0,0 +1,430 @@ +"""NetApp ONTAP adapter specifics beyond the shared contract.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters.netapp_ontap import ( + VOLUME_SUBSYSTEM_COMMENT, + NetAppOntapAdapter, + flexvol_name, + ontap_wwn, + parse_nsid, +) +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayError, + ArrayNotFoundError, + ArrayRequestError, + ArrayUnsupportedError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeError, FakeTransport +from simplyblock_core.storage_arrays.testing.netapp_ontap import ( + FakeNetAppOntap, + contract_setup, +) + +from .conftest import StaticProvider + +GIB = 1 << 30 +NQN_A = "nqn.2014-08.org.nvmexpress:uuid:host-a" +NQN_B = "nqn.2014-08.org.nvmexpress:uuid:host-b" + + +def make(**options): + setup = contract_setup(**options) + client = ArrayClient(setup.config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def iscsi(name, *iqns): + return HostSpec(name=name, initiators=tuple( + Initiator(protocol=Protocol.ISCSI, address=a) for a in iqns)) + + +def nvme(name, *nqns): + return HostSpec(name=name, initiators=tuple( + Initiator(protocol=Protocol.NVME_TCP, address=a) for a in nqns)) + + +def calls(fake: FakeNetAppOntap, method: str, fragment: str) -> list[int]: + return [i for i, (m, p) in enumerate(fake.requests) if m == method and fragment in p] + + +# -- identity helpers ----------------------------------------------------------- + + +def test_wwn_from_serial(): + # 12 ASCII chars -> 24 hex digits behind NetApp's NAA-6 prefix + assert ontap_wwn("wCVt1]IlvQWv") == "naa.600a0980" + "7743567431" + "5d496c76515776" + assert len(ontap_wwn("wCVt1]IlvQWv")) == len("naa.") + 32 + + +def test_volume_wwn_matches_fake_serial(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + serial = fake.luns[vol.id]["serial_number"] + assert vol.wwn == ontap_wwn(serial) == vol.identifier + assert vol.nguid is None + + +def test_parse_nsid_and_flexvol_name(): + assert parse_nsid("00000001h") == 1 + assert parse_nsid("0000001Ah") == 26 + assert parse_nsid(3) == 3 + assert parse_nsid(None) is None + assert flexvol_name("sb-0123abcd") == "sb_0123abcd" + + +# -- layout ------------------------------------------------------------------------ + + +def test_create_makes_thin_flexvol_then_lun(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), 10 * GIB) + lun = fake.luns[vol.id] + flex = fake.volumes[lun["location"]["volume"]["uuid"]] + assert flex["name"] == flexvol_name(vol.name) + assert lun["name"] == f"/vol/{flex['name']}/{vol.name}" + assert lun["os_type"] == "linux" + assert lun["space"]["guarantee"]["requested"] is False + assert flex["guarantee"]["type"] == "none" + assert flex["space"]["snapshot"]["reserve_percent"] == 0 + assert flex["snapshot_policy"]["name"] == "none" + assert flex["size"] >= 10 * GIB * 105 // 100 + assert calls(fake, "POST", "/api/storage/volumes")[0] < calls(fake, "POST", + "/api/storage/luns")[0] + + +def test_resize_grows_flexvol_before_lun(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), 4 * GIB) + fake.requests.clear() + grown = client.resize_volume(vol.id, 40 * GIB) + assert grown.size_bytes == 40 * GIB + vol_patch = calls(fake, "PATCH", "/api/storage/volumes/") + lun_patch = calls(fake, "PATCH", "/api/storage/luns/") + assert vol_patch and lun_patch and vol_patch[0] < lun_patch[0] + flex_uuid = fake.luns[vol.id]["location"]["volume"]["uuid"] + assert fake.volumes[flex_uuid]["size"] >= 40 * GIB + + +def test_fake_refuses_lun_larger_than_flexvol(): + """Guards the ordering above: the fake rejects a LUN that outgrows its FlexVol.""" + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), 4 * GIB) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayRequestError, match="exceeds"): + adapter._call("PATCH", f"/api/storage/luns/{vol.id}", + json={"space": {"size": 64 * GIB}}) + + +def test_delete_removes_lun_then_flexvol(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + fake.requests.clear() + assert client.delete_volume(vol.id) + assert calls(fake, "DELETE", "/api/storage/luns/")[0] < calls( + fake, "DELETE", "/api/storage/volumes/")[0] + assert not fake.luns and not fake.volumes + + +def test_shared_flexvol_mode(): + client, fake = make(volume_per_lun=False, flexvol="sb_shared") + fake.add_flexvol("sb_shared", 100 * GIB) + vol = client.create_volume(str(uuid.uuid4()), 4 * GIB) + assert fake.luns[vol.id]["name"] == f"/vol/sb_shared/{vol.name}" + assert len(fake.volumes) == 1 + client.resize_volume(vol.id, 8 * GIB) + assert not calls(fake, "PATCH", "/api/storage/volumes/") + client.delete_volume(vol.id) + assert not fake.luns and [v["name"] for v in fake.volumes.values()] == ["sb_shared"] + + +def test_rerun_reuses_flexvol_of_interrupted_create(): + client, fake = make() + vol_uuid = str(uuid.uuid4()) + name = client.volume_name(vol_uuid) + fake.add_flexvol(flexvol_name(name), 2 * GIB) + vol = client.create_volume(vol_uuid, 8 * GIB) + assert len(fake.volumes) == 1 + assert fake.volumes[fake.luns[vol.id]["location"]["volume"]["uuid"]]["size"] >= 8 * GIB + + +def test_multiple_aggregates_pick_most_free_space(): + client, fake = make(aggregates="aggr1, aggr2") + vol = client.create_volume(str(uuid.uuid4()), GIB) + flex = fake.volumes[fake.luns[vol.id]["location"]["volume"]["uuid"]] + assert [a["name"] for a in flex["aggregates"]] == ["aggr2"] + + +# -- jobs ---------------------------------------------------------------------------- + + +def test_job_is_polled_until_success(): + client, fake = make() + fake.job_polls = 4 + client.create_volume(str(uuid.uuid4()), GIB) + assert len(calls(fake, "GET", "/api/cluster/jobs/")) == 5 + + +def test_job_failure_raises_and_rolls_nothing_back(): + client, fake = make() + fake.fail_next_job("Insufficient space in aggregate aggr1.", "917830") + with pytest.raises(ArrayRequestError, match="Insufficient space"): + client.create_volume(str(uuid.uuid4()), GIB) + assert not fake.volumes and not fake.luns + + +def test_duplicate_flexvol_job_failure_is_conflict(): + client, fake = make() + adapter_vol = "sb_dup" + fake.add_flexvol(adapter_vol, GIB) + client.get_info() + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="Duplicate"): + adapter._create_flexvol(adapter_vol, GIB) + + +def test_job_timeout(): + client, fake = make(job_timeout_s=10) + fake.jobs_stuck = True + with pytest.raises(ArrayRequestError, match="still running"): + client.create_volume(str(uuid.uuid4()), GIB) + + +def test_lun_create_failure_removes_new_flexvol(monkeypatch): + client, fake = make() + + def refuse(*a, **kw): + raise FakeError(400, {"error": {"message": "LUN creation refused", "code": "1"}}) + + monkeypatch.setattr(fake, "_check_size", refuse) + with pytest.raises(ArrayRequestError, match="refused"): + client.create_volume(str(uuid.uuid4()), GIB) + assert not fake.volumes + + +# -- hosts ------------------------------------------------------------------------------- + + +def test_igroup_is_mixed_linux_and_fc_wwpn_is_colon_lowercase(): + client, fake = make() + spec = HostSpec(name="h1", initiators=( + Initiator(protocol=Protocol.ISCSI, address="IQN.2026-01.io.sb:h1"), + Initiator(protocol=Protocol.FC, address="0x21000024FF4C1A2B"))) + assert client.ensure_host(spec) == "h1" + (group,) = fake.igroups.values() + assert group["protocol"] == "mixed" and group["os_type"] == "linux" + assert {i["name"] for i in group["initiators"]} == { + "iqn.2026-01.io.sb:h1", "21:00:00:24:ff:4c:1a:2b"} + + +def test_initiator_in_foreign_igroup_conflicts_although_ontap_allows_it(): + client, fake = make() + client.get_info() + adapter = client._adapter + assert adapter is not None + # ONTAP itself accepts the same IQN in a second igroup ... + for name in ("customer", "other"): + adapter._call("POST", "/api/protocols/san/igroups", json={ + "svm": {"name": "svm1"}, "name": name, "os_type": "linux", "protocol": "iscsi", + "initiators": [{"name": "iqn.2026-01.io.sb:x"}]}) + # ... the adapter refuses + with pytest.raises(ArrayConflictError, match="customer"): + client.ensure_host(iscsi("mine", "iqn.2026-01.io.sb:x")) + assert "mine" not in {g["name"] for g in fake.igroups.values()} + + +def test_existing_iscsi_igroup_is_widened_for_fc(): + client, fake = make() + client.get_info() + adapter = client._adapter + assert adapter is not None + adapter._call("POST", "/api/protocols/san/igroups", json={ + "svm": {"name": "svm1"}, "name": "h2", "os_type": "linux", "protocol": "iscsi", + "initiators": [{"name": "iqn.2026-01.io.sb:h2"}]}) + client.ensure_host(HostSpec(name="h2", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:h2"), + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:99")))) + (group,) = fake.igroups.values() + assert group["protocol"] == "mixed" and len(group["initiators"]) == 2 + + +def test_nvme_fc_wwpn_address_is_refused(): + client, _ = make() + with pytest.raises(ArrayUnsupportedError, match="NQN"): + client.ensure_host(HostSpec(name="fc", initiators=( + Initiator(protocol=Protocol.NVME_FC, address="21:00:00:24:ff:4c:1a:2b"),))) + + +def test_nvme_rdma_is_not_supported(): + client, _ = make() + with pytest.raises(ArrayError, match="does not support"): + client.ensure_host(HostSpec(name="r", initiators=( + Initiator(protocol=Protocol.NVME_RDMA, address=NQN_A),))) + + +# -- options ------------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("options", "match"), [ + ({"svm": None}, "needs options: svm"), + ({"protocol_family": "fcoe"}, "protocol_family"), + ({"volume_per_lun": False}, "flexvol is required"), + ({"space_guarantee": "file"}, "space_guarantee"), + ({"snapshot_reserve_percent": 95}, "snapshot_reserve_percent"), + ({"namespace_block_size": 1024}, "namespace_block_size"), + ({"job_timeout_s": "soon"}, "job_timeout_s"), + ({"pool": "x"}, "Unknown NetApp ONTAP options"), +]) +def test_option_validation(options, match): + setup = contract_setup() + opts = {**setup.config.options, **options} + opts = {k: v for k, v in opts.items() if v is not None} + from simplyblock_core.storage_arrays.base import AdapterContext + from simplyblock_core.storage_arrays.http import RestSession + config = setup.config.model_copy(update={"options": opts}) + with pytest.raises(ArrayError, match=match): + NetAppOntapAdapter(AdapterContext(config, setup.credential, + RestSession(config.endpoint))) + + +def test_unknown_svm_fails_login(): + setup = contract_setup() + config = setup.config.model_copy(update={"options": {"svm": "nope"}}) + client = ArrayClient(config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake)) + with pytest.raises(ArrayNotFoundError, match="nope"): + client.get_info() + + +def test_get_info(): + client, _ = make() + info = client.get_info() + assert info.model == "AFF-A400" + assert info.version.startswith("NetApp Release 9.14.1") + assert info.serial + + +# -- NVMe family and protocol routing ------------------------------------------------------ + + +def test_nvme_family_creates_namespace(): + client, fake = make(protocol_family="nvme") + vol = client.create_volume(str(uuid.uuid4()), 8 * GIB) + assert vol.id in fake.namespaces and not fake.luns + ns = fake.namespaces[vol.id] + assert ns["space"]["block_size"] == 4096 and ns["os_type"] == "linux" + assert vol.wwn is None and vol.identifier == f"id.{vol.id}" + assert client.get_volume(vol.id) == vol + grown = client.resize_volume(vol.id, 16 * GIB, expected_identifier=vol.identifier) + assert grown.size_bytes == 16 * GIB + + +def test_nvme_multi_attach_through_volume_subsystem(): + client, fake = make(protocol_family="nvme") + vol = client.create_volume(str(uuid.uuid4()), 8 * GIB) + ha = client.ensure_host(nvme("node-a", NQN_A)) + hb = client.ensure_host(nvme("node-b", NQN_B)) + ma = client.map_volume(vol.id, ha) + assert ma.lun == 1 and ma.host_id == "node-a" + assert client.map_volume(vol.id, ha) == ma + client.map_volume(vol.id, hb) + assert {m.host_id for m in client.list_mappings(vol.id)} == {"node-a", "node-b"} + (smap,) = fake.subsystem_maps + vsub = fake.subsystems[smap["subsystem"]["uuid"]] + assert vsub["comment"] == VOLUME_SUBSYSTEM_COMMENT + assert vsub["name"] == flexvol_name(vol.name) + assert {h["nqn"] for h in vsub["hosts"]} == {NQN_A, NQN_B} + + client.unmap_volume(vol.id, ha) + client.unmap_volume(vol.id, ha) + assert [m.host_id for m in client.list_mappings(vol.id)] == ["node-b"] + + assert client.delete_volume(vol.id) + assert not fake.namespaces and not fake.subsystem_maps and not fake.volumes + assert {s["name"] for s in fake.subsystems.values()} == {"node-a", "node-b"} + client.delete_host(ha) + client.delete_host(hb) + assert not fake.subsystems + + +def test_volume_subsystem_does_not_count_as_host_owner(): + client, _ = make(protocol_family="nvme") + vol = client.create_volume(str(uuid.uuid4()), GIB) + client.map_volume(vol.id, client.ensure_host(nvme("node-a", NQN_A))) + # NQN_A is now in node-a and in the volume subsystem: re-ensuring is fine, + assert client.ensure_host(nvme("node-a", NQN_A)) == "node-a" + # but another host claiming it conflicts. + with pytest.raises(ArrayConflictError, match="node-a"): + client.ensure_host(nvme("node-x", NQN_A)) + + +def test_scsi_volume_to_nvme_host_is_unsupported_and_vice_versa(): + client, _ = make() + lun = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(nvme("nv", NQN_A)) + with pytest.raises(ArrayUnsupportedError, match="NVMe host"): + client.map_volume(lun.id, host) + + nclient, nfake = make(protocol_family="nvme") + ns = nclient.create_volume(str(uuid.uuid4()), GIB) + nclient.ensure_host(iscsi("sc", "iqn.2026-01.io.sb:sc")) + with pytest.raises(ArrayUnsupportedError, match="SCSI host"): + nclient.map_volume(ns.id, "sc") + assert not nfake.subsystem_maps + + +def test_volumes_of_the_other_family_are_still_found(): + client, fake = make(protocol_family="nvme") + ns = client.create_volume(str(uuid.uuid4()), GIB) + scsi_setup_config = client.config.model_copy( + update={"options": {"svm": "svm1", "protocol_family": "scsi"}}) + provider = StaticProvider(contract_setup().credential) + scsi_client = ArrayClient(scsi_setup_config, provider, # type: ignore[arg-type] + transport=FakeTransport(fake)) + assert scsi_client.get_volume(ns.id) == ns + assert scsi_client.delete_volume(ns.id) + assert not fake.namespaces + + +def test_namespace_mapped_to_foreign_subsystem_is_refused(): + client, fake = make(protocol_family="nvme") + vol = client.create_volume(str(uuid.uuid4()), GIB) + adapter = client._adapter + assert adapter is not None + adapter._call("POST", "/api/protocols/nvme/subsystems", json={ + "svm": {"name": "svm1"}, "name": "customer_ss", "os_type": "linux"}) + adapter._call("POST", "/api/protocols/nvme/subsystem-maps", json={ + "svm": {"name": "svm1"}, "namespace": {"uuid": vol.id}, + "subsystem": {"name": "customer_ss"}}) + client.ensure_host(nvme("node-a", NQN_A)) + with pytest.raises(ArrayConflictError, match="does not manage"): + client.map_volume(vol.id, "node-a") + + +def test_lun_numbers_come_from_lun_maps(): + client, fake = make() + host = client.ensure_host(iscsi("h", "iqn.2026-01.io.sb:h")) + v1 = client.create_volume(str(uuid.uuid4()), GIB) + v2 = client.create_volume(str(uuid.uuid4()), GIB) + assert client.map_volume(v1.id, host).lun == 0 + assert client.map_volume(v2.id, host).lun == 1 + assert {m["logical_unit_number"] for m in fake.lun_maps} == {0, 1} + + +def test_map_to_missing_host_is_not_found(): + client, _ = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + with pytest.raises(ArrayNotFoundError): + client.map_volume(vol.id, "ghost") From 144ca3790ca5eb3c3ad15b5f417d1f5ffd82ee9c Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 16:22:27 +0100 Subject: [PATCH 30/50] feat(array): SAN arrays as multi-attach plugins; credential references Plug the 11 storage_arrays adapters into the multi-attach array LUN mode of #1626 and move its array credentials onto simplyblock_core.credentials. Plugins (simplyblock_core/array/san.py): one ArrayPlugin per array family (pure-flasharray, netapp-ontap, dell-powerstore/-powermax/-powerflex, hpe-wsapi, hpe-nimble, ibm-flashsystem, hitachi-vsp, huawei-dorado, infinidat-infinibox), built in and declared as entry points. Ownership is encoded in the LUN name (sb-): SAN arrays have no tags, and every operation refuses another cluster's LUN. wwn is returned in the form lsblk reports. option `protocol` selects the host initiators. list_luns is not offered; cluster delete deletes the recorded LUNs. Host side: the agent reports its SAN initiators in /info (iSCSI IQN, FC WWPNs, NVMe host NQN, PowerFlex SDC GUID; /host/etc first on Kubernetes) and gets POST /scan_new_luns (SCSI host scan, NVMe namespace rescan). Node add and the attachment reconcile scan while they wait for a LUN; an agent without the endpoint is tolerated. WWN matching accepts naa.. Credentials: register_array no longer stores plaintext through kms.put_secret. Given credentials are wrapped by the cluster's credentials key (Vault Transit, KMIP, or the local KMS only with allow_local_kms_credentials), bound to array id + endpoint, and kept as Cluster.array_credential_ref; or credential_ref names an external Vault KV / KMIP secret that is only read. Arrays registered before keep working from their KMS path, which a re-registration removes. API/CLI: credential_ref / --credential-ref. CredentialProvider resolves the KMS connection per call. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 11 + simplyblock_cli/cli-reference.yaml | 9 +- simplyblock_cli/cli.py | 1 + simplyblock_cli/clibase.py | 10 +- simplyblock_core/array/__init__.py | 64 +++- simplyblock_core/array/san.py | 340 ++++++++++++++++++ .../controllers/array_controller.py | 163 +++++++-- simplyblock_core/credentials.py | 11 +- simplyblock_core/models/cluster.py | 11 +- simplyblock_core/multi_attach.py | 3 +- simplyblock_core/snode_client.py | 5 + .../api/internal/storage_node/docker.py | 19 + .../api/internal/storage_node/kubernetes.py | 19 + simplyblock_web/api/v2/cluster/array.py | 18 +- simplyblock_web/node_utils.py | 74 +++- tests/unit/storage_arrays/test_san_plugin.py | 154 ++++++++ tests/unit/test_array_lifecycle.py | 144 +++++++- tests/unit/test_credentials.py | 4 + tests/unit/test_san_host_agent.py | 96 +++++ 19 files changed, 1094 insertions(+), 62 deletions(-) create mode 100644 simplyblock_core/array/san.py create mode 100644 tests/unit/storage_arrays/test_san_plugin.py create mode 100644 tests/unit/test_san_host_agent.py diff --git a/pyproject.toml b/pyproject.toml index 31d6e99c70..d92f5d6759 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,17 @@ simplyblock-task-runner = "simplyblock_core.services.task_runners:main" # (simplyblock_core.array); the name is the value of Cluster.array_plugin. [project.entry-points."simplyblock.array_plugins"] aws-ebs = "simplyblock_core.array.aws_ebs:AwsEbsPlugin" +pure-flasharray = "simplyblock_core.array.san:PureFlashArrayPlugin" +netapp-ontap = "simplyblock_core.array.san:NetAppOntapPlugin" +dell-powerstore = "simplyblock_core.array.san:DellPowerStorePlugin" +dell-powermax = "simplyblock_core.array.san:DellPowerMaxPlugin" +dell-powerflex = "simplyblock_core.array.san:DellPowerFlexPlugin" +hpe-wsapi = "simplyblock_core.array.san:HpeWsapiPlugin" +hpe-nimble = "simplyblock_core.array.san:HpeNimblePlugin" +ibm-flashsystem = "simplyblock_core.array.san:IbmFlashSystemPlugin" +hitachi-vsp = "simplyblock_core.array.san:HitachiVspPlugin" +huawei-dorado = "simplyblock_core.array.san:HuaweiDoradoPlugin" +infinidat-infinibox = "simplyblock_core.array.san:InfinidatInfiniBoxPlugin" [dependency-groups] test = [ diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index d9e7a49083..9c23422597 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1596,7 +1596,8 @@ commands: usage: > simplyblock creates, multi-attaches, grows, fences and deletes the LUNs of a multi-attach cluster through the array plugin registered here. The plugin is loaded and connected with the credentials before anything - is saved. The credentials are stored in the cluster's KMS, never in the cluster record. With --lun-size, + is saved. The credentials are wrapped by the cluster's KMS (or stay in an external Vault KV / KMIP secret named + by --credential-ref), never stored in the clear. With --lun-size, every node added afterwards gets a LUN of that size created and attached on the array. arguments: - name: "cluster_id" @@ -1624,6 +1625,12 @@ commands: aws_secret_access_key; omit to use the instance role). dest: credentials_file type: str + - name: "--credential-ref" + help: > + Instead of --credentials-file: a secret kept outside simplyblock, as JSON, e.g. + `{"kind": "vault-kv", "mount": "secret", "path": "arrays/pure01"}` or `{"kind": "kmip-secret", "uid": "42"}`. + dest: credential_ref + type: str - name: "--lun-size" help: "Size of the LUN created for each node added to the cluster, e.g. `1T`. Default: none." dest: lun_size diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 68dfb20176..5f3d845cb8 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -583,6 +583,7 @@ def init_cluster__set_array(self, subparser): subcommand.add_argument('--endpoint', help='The array\'s management endpoint (for aws-ebs: an EC2 endpoint URL override, usually empty).', type=str, default='', dest='endpoint') subcommand.add_argument('--options', help='Plugin options as a JSON object, e.g. `{"az": "us-east-1a", "iops": 6000}`.', type=str, dest='options') subcommand.add_argument('--credentials-file', help='A JSON file with the array credentials as an object of strings (for aws-ebs: aws_access_key_id, aws_secret_access_key; omit to use the instance role).', type=str, dest='credentials_file') + subcommand.add_argument('--credential-ref', help='Instead of --credentials-file: a secret kept outside simplyblock, as JSON, e.g. `{"kind": "vault-kv", "mount": "secret", "path": "arrays/pure01"}` or `{"kind": "kmip-secret", "uid": "42"}`.', type=str, dest='credential_ref') subcommand.add_argument('--lun-size', help='Size of the LUN created for each node added to the cluster, e.g. `1T`. Default: none.', type=size_type(), default='0', dest='lun_size') def init_cluster__get_array(self, subparser): diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index bc4167faa1..9733aee3f7 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -879,9 +879,17 @@ def cluster__set_array(self, sub_command, args): raise ValueError("--credentials-file must hold a JSON object of strings") # Wrapped at the boundary; unwrapped only by the KMS and the plugin. credentials = {str(k): SecretStr(v) for k, v in raw.items()} + credential_ref = None + if getattr(args, "credential_ref", None): + try: + credential_ref = json.loads(args.credential_ref) + except json.JSONDecodeError as e: + raise ValueError(f"--credential-ref is not valid JSON: {e}") + if not isinstance(credential_ref, dict): + raise ValueError("--credential-ref must be a JSON object") view = array_controller.register_array( args.cluster_id, args.plugin, args.endpoint or "", options, credentials, - int(args.lun_size or 0)) + int(args.lun_size or 0), credential_ref=credential_ref) return _format_result([view], json=getattr(args, "json", False)) def cluster__get_array(self, sub_command, args): diff --git a/simplyblock_core/array/__init__.py b/simplyblock_core/array/__init__.py index cff580e013..b491a9d893 100644 --- a/simplyblock_core/array/__init__.py +++ b/simplyblock_core/array/__init__.py @@ -11,8 +11,8 @@ registry that loads a vendor plugin by name from the entry-point group ``simplyblock.array_plugins``. The array's credentials live in the KMS at ``cluster//array/``; the Cluster record keeps the plugin -name, endpoint and options only. Built-in plugin: ``aws-ebs`` -(:mod:`.aws_ebs`). Registration and the LUN lifecycle are in +name, endpoint and options only. Built-in plugins: ``aws-ebs`` +(:mod:`.aws_ebs`) and the SAN arrays (:mod:`.san`). Registration and the LUN lifecycle are in :mod:`simplyblock_core.controllers.array_controller`. Plugins are found only through the entry-point group or an explicit @@ -40,6 +40,18 @@ #: pyproject.toml). BUILTIN_PLUGINS = { "aws-ebs": "simplyblock_core.array.aws_ebs:AwsEbsPlugin", + # SAN arrays: the storage_arrays adapters (simplyblock_core.array.san). + "pure-flasharray": "simplyblock_core.array.san:PureFlashArrayPlugin", + "netapp-ontap": "simplyblock_core.array.san:NetAppOntapPlugin", + "dell-powerstore": "simplyblock_core.array.san:DellPowerStorePlugin", + "dell-powermax": "simplyblock_core.array.san:DellPowerMaxPlugin", + "dell-powerflex": "simplyblock_core.array.san:DellPowerFlexPlugin", + "hpe-wsapi": "simplyblock_core.array.san:HpeWsapiPlugin", + "hpe-nimble": "simplyblock_core.array.san:HpeNimblePlugin", + "ibm-flashsystem": "simplyblock_core.array.san:IbmFlashSystemPlugin", + "hitachi-vsp": "simplyblock_core.array.san:HitachiVspPlugin", + "huawei-dorado": "simplyblock_core.array.san:HuaweiDoradoPlugin", + "infinidat-infinibox": "simplyblock_core.array.san:InfinidatInfiniBoxPlugin", } @@ -269,10 +281,39 @@ def load_plugin(name: str, endpoint: str = "", options: dict | None = None, return plugin +def credential_context(array_id: str, endpoint: str) -> str: + """What an array's credentials are bound to: the array id and the + scheme/host/port of its endpoint. A credential wrapped for one array is + refused for another endpoint, so rewriting the endpoint in the database + cannot send it elsewhere.""" + from urllib.parse import urlsplit + ep = (endpoint or "").strip() + parts = urlsplit(ep) + if parts.scheme and parts.hostname: + port = parts.port or (443 if parts.scheme == "https" else 80) + ep = f"{parts.scheme}://{parts.hostname.lower()}:{port}" + return f"array/{array_id}/{ep}" + + +def credential_ref_view(cluster) -> dict: + """The cluster's array credential reference without anything secret: + the kind, and for an external secret where it lives.""" + ref = dict(getattr(cluster, "array_credential_ref", None) or {}) + if not ref: + return {} + if ref.get("kind") == "managed": + return {"kind": "managed", "backend": ref.get("backend", "")} + return {k: v for k, v in ref.items() if k in ("kind", "mount", "path", "version", "uid", "field")} + + def array_credentials(cluster) -> dict: - """The credentials of the cluster's array from the KMS (``SecretStr`` - values), or ``{}`` when the array was registered without any (e.g. the - EBS plugin on the instance role) or no array is registered. + """The credentials of the cluster's array (``SecretStr`` values), or + ``{}`` when the array was registered without any (e.g. the EBS plugin on + the instance role) or no array is registered. + + Read through ``Cluster.array_credential_ref`` (wrapped by the cluster's + KMS, or an external Vault KV / KMIP secret); arrays registered before + references existed are read from their KMS path. Raises: ArrayError: a secret is recorded but cannot be read. @@ -280,6 +321,16 @@ def array_credentials(cluster) -> dict: array_id = getattr(cluster, "array_id", "") or "" if not array_id or getattr(cluster, "array_has_credentials", False) is not True: return {} + ref = getattr(cluster, "array_credential_ref", None) or {} + if ref: + from simplyblock_core import credentials as creds_mod + try: + resolved = creds_mod.CredentialProvider(cluster, ttl_s=0).resolve( + ref, credential_context(array_id, getattr(cluster, "array_endpoint", "") or "")) + except creds_mod.CredentialError as e: + raise ArrayError(f"array credentials could not be read: {e}") from e + return {name: resolved.get(name) for name in resolved.field_names + if name != creds_mod.CONTEXT_FIELD} from simplyblock_core import kms try: with kms.create_kms_connection(cluster) as conn: @@ -319,7 +370,8 @@ def _config_key(cluster) -> tuple: getattr(cluster, "array_endpoint", "") or "", json.dumps(getattr(cluster, "array_options", None) or {}, sort_keys=True, default=str), getattr(cluster, "array_id", "") or "", - getattr(cluster, "array_has_credentials", False) is True) + getattr(cluster, "array_has_credentials", False) is True, + json.dumps(getattr(cluster, "array_credential_ref", None) or {}, sort_keys=True)) def cached_plugin_for_cluster(cluster, max_age: float | None = None) -> ArrayPlugin: diff --git a/simplyblock_core/array/san.py b/simplyblock_core/array/san.py new file mode 100644 index 0000000000..4950687f4f --- /dev/null +++ b/simplyblock_core/array/san.py @@ -0,0 +1,340 @@ +"""SAN array plugins: the vendor adapters of :mod:`simplyblock_core.storage_arrays` +as :class:`~simplyblock_core.array.ArrayPlugin` s. + +One plugin per supported array family (``pure-flasharray``, ``netapp-ontap``, +... see :data:`SAN_PLUGINS`). Each wraps an +:class:`~simplyblock_core.storage_arrays.client.ArrayClient`, which already +gives the plugin contract its guarantees: idempotent create / map / unmap / +delete, grow-only resize, one re-authentication on a rejected session. + +Ownership. SAN arrays mostly have no free-form tags, so a LUN's owner is +encoded in its name: ``sb-`` (27 characters, +inside every supported array's name limit). The cluster part makes every +operation refuse a LUN of another cluster (:class:`ForeignLunError`); the LUN +part is derived from the name the control plane asked for, so a retried +create finds the LUN it made before. + +Hosts. A storage node is registered with the initiators of the protocol named +by the ``protocol`` option (``iscsi``, ``fc``, ``nvme-tcp``, ``nvme-rdma``, +``nvme-fc``, ``sdc``), taken from what its agent reports +(``host_initiators``: ``iscsi_iqn``, ``fc_wwpns``, ``nvme_hostnqn``, +``sdc_guid``). The host object is named ``sb-`` (shortened with a +hash suffix where the array's limit is shorter). + +Identity. ``wwn`` is returned in the form the hosts report it (lsblk ``WWN``): +the NAA hex digits for SCSI, ``eui.`` for NVMe namespaces. + +Options (``Cluster.array_options``): ``protocol`` (required), ``ca_bundle`` +(PEM), ``verify_tls`` (default true), ``timeout_s``, and the adapter's own +options (``option_docs`` of the adapter: pool, SVM, CPG, port group, ...). +``host_portals`` (iSCSI / NVMe discovery portals the nodes log in to) is read +by the node-side scan, not here. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +import uuid as uuid_mod +from typing import Any + +from pydantic import SecretStr, ValidationError + +from simplyblock_core.array import ArrayError, ArrayPlugin, ForeignLunError +from simplyblock_core.credentials import Credential +from simplyblock_core.storage_arrays import base as sa +from simplyblock_core.storage_arrays.client import MANAGED_PREFIX, ArrayClient + +logger = logging.getLogger() + +#: Options consumed by the plugin itself; everything else goes to the adapter. +_PLUGIN_OPTIONS = frozenset({"protocol", "ca_bundle", "verify_tls", "timeout_s", + "host_portals"}) + +#: ``host_initiators`` keys per protocol. +_INITIATOR_KEYS = { + sa.Protocol.ISCSI: "iscsi_iqn", + sa.Protocol.FC: "fc_wwpns", + sa.Protocol.NVME_TCP: "nvme_hostnqn", + sa.Protocol.NVME_RDMA: "nvme_hostnqn", + sa.Protocol.NVME_FC: "nvme_hostnqn", + sa.Protocol.SDC: "sdc_guid", +} + +_CLUSTER_HEX = 8 +_LUN_HEX = 16 + + +def _hex(text: str, n: int) -> str: + return hashlib.sha256(text.encode()).hexdigest()[:n] + + +def lun_uuid(cluster_id: str, name: str) -> str: + """The volume UUID whose managed name encodes cluster and LUN name.""" + hex32 = _hex(f"cluster:{cluster_id}", _CLUSTER_HEX) + _hex(f"lun:{cluster_id}/{name}", 24) + return str(uuid_mod.UUID(hex32)) + + +def cluster_prefix(cluster_id: str) -> str: + return MANAGED_PREFIX + _hex(f"cluster:{cluster_id}", _CLUSTER_HEX) + + +def host_wwn(vol: sa.Volume, protocol: sa.Protocol) -> str: + """The identity a host reports for ``vol`` (lsblk ``WWN``).""" + if protocol.is_nvme and vol.nguid: + return vol.nguid.lower() + if vol.wwn: + w = vol.wwn.lower() + return w.removeprefix("naa.") + return vol.identifier.lower() + + +def host_object_name(hostname: str, max_len: int) -> str: + clean = re.sub(r"[^A-Za-z0-9-]", "-", hostname or "").strip("-") or "host" + name = f"{MANAGED_PREFIX}{clean}" + if len(name) <= max_len: + return name + suffix = "-" + _hex(hostname, 8) + return name[: max_len - len(suffix)].rstrip("-") + suffix + + +class _StaticCredentials: + """The plugin gets its credentials already resolved (from the KMS).""" + + def __init__(self, fields: dict[str, SecretStr]): + self._credential = Credential(fields) + + def resolve(self, ref, context): + return self._credential + + def invalidate(self, ref=None): + pass + + +def _as_list(value) -> list[str]: + if not value: + return [] + if isinstance(value, str): + return [v for v in re.split(r"[\s,]+", value) if v] + return [str(v) for v in value if v] + + +class SanArrayPlugin(ArrayPlugin): + """Base of the per-vendor plugins; ``vendor`` is the adapter key.""" + + vendor: str = "" + + def __init__(self, endpoint: str = "", options: dict | None = None, credentials=None, + cluster_id: str = ""): + super().__init__(endpoint, options, credentials, cluster_id) + self.client: ArrayClient | None = None + #: Tests: a ``requests`` transport (vendor fake) and session kwargs. + self.transport: Any = None + self.session_kwargs: dict[str, Any] = {} + + # ------------------------------------------------------------ setup + + @property + def protocol(self) -> sa.Protocol: + raw = str(self.options.get("protocol") or "").strip().lower() + if not raw: + raise ArrayError(f"array plugin {self.vendor}: option 'protocol' is required " + f"({', '.join(p.value for p in sa.Protocol)})") + try: + return sa.Protocol(raw) + except ValueError: + raise ArrayError(f"array plugin {self.vendor}: unknown protocol {raw!r}") from None + + def connect(self, endpoint: str, credentials: dict, options: dict) -> None: + super().connect(endpoint, credentials, options) + if not self.cluster_id: + raise ArrayError("the plugin has no cluster id") + proto = self.protocol + adapter_opts = {k: v for k, v in self.options.items() if k not in _PLUGIN_OPTIONS} + try: + config = sa.ArrayConfig( + id=f"{self.cluster_id}", + vendor=self.vendor, + endpoint=endpoint, + # The plugin resolves nothing itself: the provider below hands + # back the credentials it was given. + credential_ref={"kind": "vault-kv", "mount": "-", "path": "-"}, + ca_bundle=self.options.get("ca_bundle") or None, + verify_tls=_truthy(self.options.get("verify_tls", True)), + timeout_s=float(self.options.get("timeout_s") or 60.0), + options=adapter_opts, + ) + except ValidationError as e: + raise ArrayError(f"array plugin {self.vendor}: {_first_error(e)}") from None + fields = {k: v if isinstance(v, SecretStr) else SecretStr(str(v)) + for k, v in (credentials or {}).items()} + self.client = ArrayClient(config, _StaticCredentials(fields), # type: ignore[arg-type] + transport=self.transport, session_kwargs=self.session_kwargs) + caps = self.client.capabilities + if proto not in caps.protocols: + raise ArrayError(f"{self.client.adapter_cls.display_name} does not support " + f"{proto.value} (supported: " + f"{', '.join(sorted(p.value for p in caps.protocols))})") + # Validates credential shape and options, logs in. + self._run(lambda c: c.get_info()) + + def _client(self) -> ArrayClient: + if self.client is None: + raise ArrayError("array plugin is not connected") + return self.client + + def _run(self, fn): + try: + return fn(self._client()) + except ArrayError: + raise + except sa.ArrayError as e: + raise ArrayError(str(e)) from e + + def capabilities(self) -> dict: + caps = self._client().capabilities if self.client else None + protocols = sorted(p.value for p in caps.protocols) if caps else [] + return {"protocols": protocols, "supports_reservations": False, "max_attachments": 0} + + # ---------------------------------------------------------- ownership + + def _prefix(self) -> str: + return cluster_prefix(self.cluster_id) + + def _owned_volume(self, lun_id: str) -> sa.Volume | None: + vol = self._run(lambda c: c.get_volume(lun_id)) + if vol is not None and not vol.name.startswith(self._prefix()): + raise ForeignLunError( + f"LUN {lun_id} ({vol.name}) was not created by simplyblock cluster " + f"{self.cluster_id}; it is not touched") + return vol + + def _view(self, vol: sa.Volume) -> dict: + hosts = [m.host_id for m in self._run(lambda c: c.list_mappings(vol.id))] + return {"id": vol.id, "wwn": host_wwn(vol, self.protocol), "size": vol.size_bytes, + "name": vol.name, "hosts": hosts} + + # --------------------------------------------------------- operations + + def ensure_host(self, name: str, initiators: dict) -> str: + proto = self.protocol + key = _INITIATOR_KEYS[proto] + addresses = _as_list((initiators or {}).get(key)) + if not addresses: + raise ArrayError( + f"host {name!r} reports no {proto.value} initiator ({key}); check the " + "host's initiator setup (iSCSI initiatorname, FC HBA, NVMe hostnqn, SDC)") + try: + spec = sa.HostSpec( + name=host_object_name(name, self._client().capabilities.max_name_len), + initiators=tuple(sa.Initiator(protocol=proto, address=a) for a in addresses)) + except ValidationError as e: + raise ArrayError(f"host {name!r}: {_first_error(e)}") from None + return self._run(lambda c: c.ensure_host(spec)) + + def create_lun(self, name: str, size_bytes: int, tags: dict) -> dict: + self.cluster_tags(tags, name) # refuses without a cluster id + if int(size_bytes) <= 0: + raise ArrayError("a LUN needs a size") + vol = self._run(lambda c: c.create_volume(lun_uuid(self.cluster_id, name), int(size_bytes))) + return {"id": vol.id, "wwn": host_wwn(vol, self.protocol), "size": vol.size_bytes} + + def attach(self, lun_id: str, host_id: str) -> None: + if self._owned_volume(lun_id) is None: + raise ArrayError(f"LUN {lun_id} not found") + self._run(lambda c: c.map_volume(lun_id, host_id)) + + def detach(self, lun_id: str, host_id: str, force: bool = False) -> None: + if self._owned_volume(lun_id) is None: + return + # An unmap at the array takes effect for the host at once, queued + # I/O included: the same with and without ``force``. + self._run(lambda c: c.unmap_volume(lun_id, host_id)) + + def resize(self, lun_id: str, size_bytes: int) -> None: + if self._owned_volume(lun_id) is None: + raise ArrayError(f"LUN {lun_id} not found") + self._run(lambda c: c.resize_volume(lun_id, int(size_bytes))) + + def get(self, lun_id: str) -> dict: + vol = self._owned_volume(lun_id) + if vol is None: + raise ArrayError(f"LUN {lun_id} not found") + return self._view(vol) + + def delete_lun(self, lun_id: str) -> None: + if self._owned_volume(lun_id) is None: + return + self._run(lambda c: c.delete_volume(lun_id)) + + def list_luns(self) -> list[dict]: + """Not offered by the SAN adapters (no portable prefix listing); + ``array_controller.delete_cluster_luns`` deletes the recorded LUNs.""" + raise NotImplementedError(f"{self.vendor}: listing LUNs is not supported") + + +def _truthy(v) -> bool: + if isinstance(v, bool): + return v + return str(v).strip().lower() not in ("0", "false", "no", "off", "") + + +def _first_error(e: ValidationError) -> str: + err = e.errors()[0] + loc = ".".join(str(p) for p in err.get("loc", ())) + return f"{loc}: {err.get('msg')}" if loc else str(err.get("msg")) + + +class PureFlashArrayPlugin(SanArrayPlugin): + vendor = "pure-flasharray" + + +class NetAppOntapPlugin(SanArrayPlugin): + vendor = "netapp-ontap" + + +class DellPowerStorePlugin(SanArrayPlugin): + vendor = "dell-powerstore" + + +class DellPowerMaxPlugin(SanArrayPlugin): + vendor = "dell-powermax" + + +class DellPowerFlexPlugin(SanArrayPlugin): + vendor = "dell-powerflex" + + +class HpeWsapiPlugin(SanArrayPlugin): + vendor = "hpe-wsapi" + + +class HpeNimblePlugin(SanArrayPlugin): + vendor = "hpe-nimble" + + +class IbmFlashSystemPlugin(SanArrayPlugin): + vendor = "ibm-flashsystem" + + +class HitachiVspPlugin(SanArrayPlugin): + vendor = "hitachi-vsp" + + +class HuaweiDoradoPlugin(SanArrayPlugin): + vendor = "huawei-dorado" + + +class InfinidatInfiniBoxPlugin(SanArrayPlugin): + vendor = "infinidat-infinibox" + + +#: Plugin name (= adapter key) -> import path, merged into BUILTIN_PLUGINS. +SAN_PLUGINS = { + cls.vendor: f"{__name__}:{cls.__name__}" + for cls in (PureFlashArrayPlugin, NetAppOntapPlugin, DellPowerStorePlugin, + DellPowerMaxPlugin, DellPowerFlexPlugin, HpeWsapiPlugin, HpeNimblePlugin, + IbmFlashSystemPlugin, HitachiVspPlugin, HuaweiDoradoPlugin, + InfinidatInfiniBoxPlugin) +} diff --git a/simplyblock_core/controllers/array_controller.py b/simplyblock_core/controllers/array_controller.py index 7619b88072..01956e0ae6 100644 --- a/simplyblock_core/controllers/array_controller.py +++ b/simplyblock_core/controllers/array_controller.py @@ -121,6 +121,7 @@ def array_view(cluster, capabilities=None) -> dict: "options": dict(cluster.array_options or {}), "lun_size": int(cluster.array_lun_size or 0), "has_credentials": bool(cluster.array_has_credentials), + "credential_ref": array.credential_ref_view(cluster), } if capabilities is not None: out["capabilities"] = capabilities @@ -128,21 +129,33 @@ def array_view(cluster, capabilities=None) -> dict: def register_array(cluster_id, plugin, endpoint="", options=None, credentials=None, - lun_size=0) -> dict: + lun_size=0, credential_ref=None) -> dict: """Register (or re-register, e.g. to rotate credentials) the storage array of a multi-attach cluster. - The plugin is loaded and connected with the given credentials and its - capabilities read before anything is saved. Credentials (a flat dict of - strings / SecretStr) are stored in the KMS; the Cluster record keeps the - plugin name, endpoint, options and LUN size. Returns the array view - (``array_view``) with the plugin's capabilities. + The plugin is loaded and connected with the credentials and its + capabilities read before anything is saved. The Cluster record keeps the + plugin name, endpoint, options, LUN size and a credential reference + (``Cluster.array_credential_ref``), never a credential: + + * ``credentials`` (a flat dict of strings / SecretStr) are wrapped by the + cluster's credentials key in its KMS (Vault Transit or KMIP; the local + KMS only with ``allow_local_kms_credentials``), bound to the array id + and endpoint, and only the wrapped form is kept; + * ``credential_ref`` names a secret kept outside simplyblock instead + (``{"kind": "vault-kv", "mount", "path"[, "version"]}`` or + ``{"kind": "kmip-secret", "uid"[, "field"]}``): simplyblock reads it + when it connects and never stores it. + + Returns the array view (``array_view``) with the plugin's capabilities. Raises: - ValueError: not a multi-attach cluster, the plugin cannot be loaded - or connected, its capabilities do not fit the cluster, or LUNs - created through another plugin exist. + ValueError: not a multi-attach cluster, both or a bad credential + source, the credentials cannot be stored or read, the plugin + cannot be loaded or connected, its capabilities do not fit the + cluster, or LUNs created through another plugin exist. """ + from simplyblock_core import credentials as creds_mod from simplyblock_core import kms db = _db() cluster = db.get_cluster_by_id(cluster_id) @@ -153,12 +166,32 @@ def register_array(cluster_id, plugin, endpoint="", options=None, credentials=No if int(lun_size or 0) < 0: raise ValueError("lun_size must not be negative") creds = _wrap_credentials(credentials) + if creds and credential_ref: + raise ValueError("give the credentials or a reference to them, not both") opts = dict(options or {}) nodes = db.get_storage_nodes_by_cluster_id(cluster_id) if (cluster.array_plugin and cluster.array_plugin != plugin and _managed_devices(nodes)): raise ValueError( f"LUNs created through plugin {cluster.array_plugin!r} exist; the plugin cannot change") + array_id = cluster.array_id or str(uuid_mod.uuid4()) + context = array.credential_context(array_id, endpoint or "") + ref: dict = {} + if credential_ref: + try: + parsed = creds_mod.CREDENTIAL_REF_ADAPTER.validate_python(dict(credential_ref)) + except Exception as e: # noqa: BLE001 - pydantic + raise ValueError(f"credential_ref is not a valid reference: {e}") from None + if isinstance(parsed, creds_mod.ManagedCredentialRef): + raise ValueError("credential_ref names an external secret (vault-kv or kmip-secret); " + "give the credentials themselves to have them stored") + ref = parsed.model_dump() + try: + resolved = creds_mod.CredentialProvider(cluster, ttl_s=0).resolve(parsed, context) + except creds_mod.CredentialError as e: + raise ValueError(f"the referenced credentials could not be read: {e}") from None + creds = {name: resolved.get(name) for name in resolved.field_names + if name != creds_mod.CONTEXT_FIELD} try: inst = array.load_plugin(plugin, endpoint, opts, credentials=creds, cluster_id=cluster_id) caps = dict(inst.capabilities() or {}) @@ -172,32 +205,32 @@ def register_array(cluster_id, plugin, endpoint="", options=None, credentials=No if max_att and max_att < needed: raise ValueError(f"the array presents a LUN to at most {max_att} hosts; the cluster needs {needed}") - array_id = cluster.array_id or str(uuid_mod.uuid4()) - path = kms.array_secret_path(cluster_id, array_id) - with kms.create_kms_connection(cluster) as conn: - if creds: - conn.put_secret(path, creds) - elif cluster.array_has_credentials: - conn.delete_secret(path) + if creds and not ref: + try: + ref = creds_mod.CredentialProvider(cluster).store(creds, context).model_dump() + except creds_mod.CredentialError as e: + raise ValueError(f"the array credentials could not be stored: {e}") from None + # A secret an earlier registration kept at the KMS path (before + # credential references) is removed once the new registration is saved. + legacy_secret = bool(cluster.array_has_credentials) and not cluster.array_credential_ref def mutate(fresh): fresh.array_id = array_id fresh.array_plugin = plugin fresh.array_endpoint = endpoint or "" fresh.array_options = opts - fresh.array_has_credentials = bool(creds) + fresh.array_credential_ref = ref + fresh.array_has_credentials = bool(ref) fresh.array_lun_size = int(lun_size or 0) - try: - db.atomic_update(cluster, mutate) - except Exception: - if creds and not cluster.array_has_credentials: - try: - with kms.create_kms_connection(cluster) as conn: - conn.delete_secret(path) - except Exception: # noqa: BLE001 - logger.exception("could not remove the array secret after a failed registration") - raise + db.atomic_update(cluster, mutate) + if legacy_secret: + try: + with kms.create_kms_connection(cluster) as conn: + conn.delete_secret(kms.array_secret_path(cluster_id, array_id)) + except Exception: # noqa: BLE001 + logger.exception("could not remove the array's earlier KMS secret") + array.forget_plugin(cluster_id) cluster = db.get_cluster_by_id(cluster_id) logger.info("Cluster %s: array %s registered (plugin %s, endpoint %s)", cluster_id, array_id, plugin, endpoint or "-") @@ -231,7 +264,9 @@ def delete_array(cluster_id, array_id, force=False) -> None: if managed and not force: raise ValueError(f"{len(managed)} LUN(s) created through the array exist; " "takeovers would lose their fence") - if cluster.array_id and cluster.array_has_credentials: + # A credential reference needs no cleanup: a managed one exists only in + # the record, an external one belongs to whoever created it. + if cluster.array_id and cluster.array_has_credentials and not cluster.array_credential_ref: with kms.create_kms_connection(cluster) as conn: conn.delete_secret(kms.array_secret_path(cluster_id, cluster.array_id)) @@ -240,6 +275,7 @@ def mutate(fresh): fresh.array_plugin = "" fresh.array_endpoint = "" fresh.array_options = {} + fresh.array_credential_ref = {} fresh.array_has_credentials = False fresh.array_lun_size = 0 @@ -249,11 +285,23 @@ def mutate(fresh): # ------------------------------------------------------------------- hosts -def host_initiators(hostname="", cloud_instance=None, system_id="", ip="") -> dict: - """What identifies a host on an array: its hostname, system id, IP and, - on AWS, the EC2 instance id (reported by the node agent from the - instance metadata service when its cloud probe runs).""" +#: SAN initiator keys the node agent reports (``san_initiators`` of its +#: /info; node_utils.get_san_initiators) and SAN plugins read. +SAN_INITIATOR_KEYS = ("iscsi_iqn", "fc_wwpns", "nvme_hostnqn", "sdc_guid") + + +def host_initiators(hostname="", cloud_instance=None, system_id="", ip="", + san_initiators=None) -> dict: + """What identifies a host on an array: its hostname, system id, IP, its + SAN initiators (iSCSI IQN, FC WWPNs, NVMe host NQN, PowerFlex SDC GUID, + as its agent reports them) and, on AWS, the EC2 instance id (reported by + the node agent from the instance metadata service when its cloud probe + runs).""" out = {"hostname": hostname} + for key in SAN_INITIATOR_KEYS: + value = (san_initiators or {}).get(key) + if value: + out[key] = value if ip: out["ip"] = ip if system_id: @@ -264,10 +312,21 @@ def host_initiators(hostname="", cloud_instance=None, system_id="", ip="") -> di return out +def _agent_san_initiators(node) -> dict: + """The SAN initiators the node's agent reports now ({} when it cannot be + asked, or is older than the report).""" + try: + info, _ = node.client(timeout=15, retry=1).info() + except Exception as e: # noqa: BLE001 + logger.warning("Node %s: SAN initiators not read from its agent: %s", node.get_id(), e) + return {} + return dict((info or {}).get("san_initiators") or {}) + + def _node_initiators(node) -> dict: cloud = {"cloud": getattr(node, "cloud_name", ""), "id": getattr(node, "cloud_instance_id", "")} return host_initiators(multi_attach.array_host(node), cloud, getattr(node, "system_uuid", ""), - getattr(node, "mgmt_ip", "") or "") + getattr(node, "mgmt_ip", "") or "", _agent_san_initiators(node)) def ensure_node_host(plugin, node, db=None) -> str: @@ -324,7 +383,8 @@ def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configur try: plugin = array.plugin_for_cluster(cluster) host_id = plugin.ensure_host(hostname, host_initiators( - hostname, node_info.get("cloud_instance"), node_info.get("system_id", ""), node_ip)) + hostname, node_info.get("cloud_instance"), node_info.get("system_id", ""), node_ip, + node_info.get("san_initiators"))) existing = next((e for e in lblk_configured or [] if e.get("array_lun_id")), None) if existing: lun = {"id": existing["array_lun_id"], "wwn": existing.get("lun_wwn", "")} @@ -342,6 +402,7 @@ def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configur deadline = time.monotonic() + timeout while True: + _scan_new_luns(snode_api) inventory, err = snode_api.get_blockdevices() found = _find_in_inventory(inventory, lun["wwn"]) if found: @@ -386,12 +447,26 @@ def add_lun_to_config(lblk_configured, entry) -> list: # ------------------------------------------------------------- attachments +def _scan_new_luns(snode_api) -> None: + """Have the host look for a LUN just presented to it. A SAN host sees a + new LUN only after a SCSI / NVMe rescan (an EBS attachment shows up by + itself, so a failure, or an agent without the endpoint, is not fatal: + the caller keeps polling).""" + try: + _, err = snode_api.scan_new_luns() + if err: + logger.debug("scan_new_luns: %s", err) + except Exception as e: # noqa: BLE001 + logger.debug("scan_new_luns: %s", e) + + def _lun_path_on(node, dev, timeout): """The path of the device's LUN on ``node`` from its agent, or "".""" owner_path = (dev.node_paths or {}).get(dev.node_id, {}).get("path", "") deadline = time.monotonic() + timeout while True: try: + _scan_new_luns(node.client(timeout=15, retry=1)) entries, _ = node.client(timeout=15, retry=1).lun_info(dev.lun_wwn) except Exception as e: # noqa: BLE001 logger.warning("lun_info on %s: %s", node.get_id(), e) @@ -711,12 +786,26 @@ def delete_cluster_luns(cluster, force=False) -> None: if not array_configured(cluster): return errors = [] + # The LUNs recorded on the cluster's devices, plus whatever the plugin + # lists (SAN plugins cannot list: their recorded LUNs are deleted). + recorded = [d.array_lun_id for _, d in + _managed_devices(_db().get_storage_nodes_by_cluster_id(cluster.get_id()))] + luns: list = [] try: plugin = array.plugin_for_cluster(cluster) - luns = plugin.list_luns() except Exception as e: # noqa: BLE001 - plugin, luns = None, [] + plugin = None errors.append(f"array: {e}") + if plugin is not None: + try: + luns = list(plugin.list_luns()) + except NotImplementedError: + logger.info("Cluster %s: the array plugin cannot list LUNs; deleting the %d recorded", + cluster.get_id(), len(recorded)) + except Exception as e: # noqa: BLE001 + errors.append(f"array: {e}") + listed = {lun["id"] for lun in luns} + luns += [{"id": lun_id} for lun_id in dict.fromkeys(recorded) if lun_id not in listed] for lun in luns: try: plugin.delete_lun(lun["id"]) # type: ignore[union-attr] @@ -727,7 +816,7 @@ def delete_cluster_luns(cluster, force=False) -> None: raise ValueError(f"array LUNs of the cluster could not be deleted: {'; '.join(errors)}") if errors: logger.error("Cluster %s deleted with array LUNs left behind: %s", cluster.get_id(), errors) - if cluster.array_id and cluster.array_has_credentials: + if cluster.array_id and cluster.array_has_credentials and not cluster.array_credential_ref: from simplyblock_core import kms try: with kms.create_kms_connection(cluster) as conn: diff --git a/simplyblock_core/credentials.py b/simplyblock_core/credentials.py index b65103acf2..066fcf2a1b 100644 --- a/simplyblock_core/credentials.py +++ b/simplyblock_core/credentials.py @@ -51,7 +51,6 @@ KMIPKMSSettings, KMSException, VaultKMSSettings, - create_kms_connection, ) if TYPE_CHECKING: @@ -158,7 +157,7 @@ def __init__( *, ttl_s: float = 300.0, clock: Callable[[], float] = time.monotonic, - kms_factory: Callable[..., KMS] = create_kms_connection, + kms_factory: Callable[..., KMS] | None = None, ): self._cluster = cluster self._ttl_s = ttl_s @@ -182,8 +181,14 @@ def _open_kms(self) -> KMS: settings = self._cluster.get_kms_settings() if isinstance(settings, VaultKMSSettings): cert_role = settings.credentials_cert_role + factory = self._kms_factory + if factory is None: + # Looked up per call, not bound at import: the KMS connection is + # replaced in tests and by callers that patch the package. + from simplyblock_core import kms as kms_pkg + factory = kms_pkg.create_kms_connection try: - return self._kms_factory(self._cluster, cert_role=cert_role) + return factory(self._cluster, cert_role=cert_role) except KMSException as e: raise CredentialError(f"Cannot connect to the KMS: {e.message}") from e diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index ae0c5f40a5..e6b2573fdb 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -236,8 +236,8 @@ def is_topology_owned(self) -> bool: # simplyblock.array_plugins, see simplyblock_core.array). Empty = no # plugin: a takeover is refused (never reopen a LUN unfenced). The array # is registered through controllers/array_controller (POST - # /clusters/{id}/arrays); its credentials live in the KMS at - # cluster//array/, never here. + # /clusters/{id}/arrays). Its credentials are never kept here, only a + # reference to them (array_credential_ref). array_plugin: str = "" array_endpoint: str = "" array_options: dict = default_factory(dict) @@ -245,6 +245,13 @@ def is_topology_owned(self) -> bool: # registered) and whether credentials were stored for it in the KMS. array_id: str = "" array_has_credentials: bool = False + # Where the array's credentials are (simplyblock_core.credentials): a + # "managed" reference (wrapped by the cluster's credentials key in its + # KMS, bound to array id + endpoint), or an external "vault-kv" / + # "kmip-secret" secret. {} with array_has_credentials = an array + # registered before references existed: its secret is in the KMS at + # cluster//array/. + array_credential_ref: dict = default_factory(dict) # Size in bytes of the LUN created for each node added to the cluster # while an array is registered; 0 = node adds create no LUN (the LUNs are # presented by the operator, as in phase 2/3). diff --git a/simplyblock_core/multi_attach.py b/simplyblock_core/multi_attach.py index ac17b752d5..756ac8c65f 100644 --- a/simplyblock_core/multi_attach.py +++ b/simplyblock_core/multi_attach.py @@ -325,8 +325,9 @@ def served_by_online_node(owner, nodes_by_id) -> list: def normalize_wwn(value) -> str: + # lsblk reports a SCSI WWN as 0x, arrays as naa.; the same id. v = (value or "").strip().lower() - return v.removeprefix("0x") + return v.removeprefix("0x").removeprefix("naa.") def lun_keys(device) -> set: diff --git a/simplyblock_core/snode_client.py b/simplyblock_core/snode_client.py index d4d806dcde..be0adc6ff0 100644 --- a/simplyblock_core/snode_client.py +++ b/simplyblock_core/snode_client.py @@ -263,6 +263,11 @@ def lun_info(self, wwn): Returns ``(result, error)``.""" return self._request("POST", "lun_info", {"wwn": wwn}) + def scan_new_luns(self): + """Have the host look for LUNs newly presented by a storage array. + Returns ``(result, error)``.""" + return self._request("POST", "scan_new_luns") + def rescan_block_device(self, device_path): """Re-read the size of a grown whole-disk block device (lblk). diff --git a/simplyblock_web/api/internal/storage_node/docker.py b/simplyblock_web/api/internal/storage_node/docker.py index b39d10cdc6..98d4403b6f 100644 --- a/simplyblock_web/api/internal/storage_node/docker.py +++ b/simplyblock_web/api/internal/storage_node/docker.py @@ -697,6 +697,22 @@ def rescan_block_device(body: _RescanBlockDeviceParams): return utils.get_response(None, str(e)) +@api.post('/scan_new_luns', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'object', + 'properties': {'scanned': {'type': 'array', 'items': {'type': 'string'}}}, + })}}}, +}) +def scan_new_luns(): + """Look for LUNs / namespaces a storage array newly presented to this + host (SCSI host scan, NVMe namespace rescan). Existing devices are not + touched.""" + try: + return utils.get_response(node_utils.scan_for_new_luns()) + except OSError as e: + return utils.get_response(None, str(e)) + + class _DeviceSmartQuery(BaseModel): device_path: str @@ -788,6 +804,9 @@ def get_info(): "network_interface": core_utils.get_nics_data(), "cloud_instance": node_info["cloud_info"], + # SAN initiators (IQN, FC WWPNs, NVMe host NQN, SDC GUID): how a storage + # array knows this host (simplyblock_core.array.san). + "san_initiators": node_utils.get_san_initiators(), "lsblk": get_node_lsblk(), "nodes_config": get_nodes_config(), diff --git a/simplyblock_web/api/internal/storage_node/kubernetes.py b/simplyblock_web/api/internal/storage_node/kubernetes.py index 7bf55d9bbd..64b1c8ae41 100644 --- a/simplyblock_web/api/internal/storage_node/kubernetes.py +++ b/simplyblock_web/api/internal/storage_node/kubernetes.py @@ -316,6 +316,9 @@ def get_info(): "host_ip": os.environ.get("HOST_IP", ""), "cloud_instance": node_info["cloud_info"], + # SAN initiators (IQN, FC WWPNs, NVMe host NQN, SDC GUID): how a storage + # array knows this host (simplyblock_core.array.san). + "san_initiators": node_utils.get_san_initiators(), "nodes_config": get_nodes_config(), }) @@ -390,6 +393,22 @@ def rescan_block_device(body: _RescanBlockDeviceParams): return utils.get_response(None, str(e)) +@api.post('/scan_new_luns', responses={ + 200: {'content': {'application/json': {'schema': utils.response_schema({ + 'type': 'object', + 'properties': {'scanned': {'type': 'array', 'items': {'type': 'string'}}}, + })}}}, +}) +def scan_new_luns(): + """Look for LUNs / namespaces a storage array newly presented to this + host (SCSI host scan, NVMe namespace rescan). Existing devices are not + touched.""" + try: + return utils.get_response(node_utils.scan_for_new_luns()) + except OSError as e: + return utils.get_response(None, str(e)) + + class _DeviceSmartQuery(BaseModel): device_path: str diff --git a/simplyblock_web/api/v2/cluster/array.py b/simplyblock_web/api/v2/cluster/array.py index 1651cf095c..ef6afd5856 100644 --- a/simplyblock_web/api/v2/cluster/array.py +++ b/simplyblock_web/api/v2/cluster/array.py @@ -1,9 +1,10 @@ """The storage array of a multi-attach cluster (array LUN mode, phase 4). simplyblock creates, presents, grows, fences and deletes the cluster's LUNs -through the array plugin registered here. Credentials go to the KMS -(cluster//array/) and are never returned; everything else is -kept on the cluster record. See simplyblock_core.controllers.array_controller. +through the array plugin registered here. Credentials are wrapped by +the cluster's KMS (or stay in an external Vault KV / KMIP secret the request +references) and are never returned; everything else is kept on the cluster +record. See simplyblock_core.controllers.array_controller. """ from typing import Annotated, Any @@ -23,8 +24,13 @@ class ArrayParams(BaseModel): plugin: Annotated[str, Field(min_length=1, description="Plugin name (entry point in simplyblock.array_plugins), e.g. aws-ebs")] endpoint: str = "" options: dict[str, Any] = Field(default_factory=dict) - # Stored in the KMS only; masked in every representation of this model. + # Wrapped by the cluster's KMS, never stored in the clear; masked in every + # representation of this model. credentials: dict[str, SecretStr] = Field(default_factory=dict) + credential_ref: Annotated[dict[str, Any], Field(description=( + "Instead of credentials: a secret kept outside simplyblock, " + '{"kind": "vault-kv", "mount": ..., "path": ..., "version": ...} or ' + '{"kind": "kmip-secret", "uid": ..., "field": ...}'))] = Field(default_factory=dict) lun_size: Annotated[util.Size, Field(description="Size of the LUN created for each added node; 0 = none")] = 0 @@ -35,6 +41,7 @@ class ArrayDTO(BaseModel): options: dict[str, Any] lun_size: int has_credentials: bool + credential_ref: dict[str, Any] = Field(default_factory=dict) capabilities: dict[str, Any] | None = None @@ -46,7 +53,8 @@ def register(cluster: Cluster, parameters: ArrayParams) -> ArrayDTO: try: view = array_controller.register_array( cluster.get_id(), parameters.plugin, parameters.endpoint, parameters.options, - parameters.credentials, parameters.lun_size) + parameters.credentials, parameters.lun_size, + credential_ref=parameters.credential_ref or None) except ValueError as e: raise HTTPException(422, str(e)) return ArrayDTO(**view) diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index f973eed7a8..ad0a5a93b9 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -453,8 +453,80 @@ def simplyblock_data_signature(device_name: str) -> bool | None: def _normalize_wwn(value: str) -> str: + # lsblk reports a SCSI WWN as 0x, arrays as naa.; the same id. v = (value or "").strip().lower() - return v.removeprefix("0x") + return v.removeprefix("0x").removeprefix("naa.") + + +def _host_file(path: str) -> str: + """Contents of a host configuration file: the host's /etc as mounted into + the Kubernetes agent (/host/etc) first, then the local one (Docker agent + on the host).""" + for candidate in (f"/host{path}", path): + text = _read_sysfs(candidate) + if text: + return text + return "" + + +def get_san_initiators() -> dict: + """The host's SAN initiator identities, for registering it on a storage + array: ``iscsi_iqn`` (/etc/iscsi/initiatorname.iscsi), ``fc_wwpns`` (FC + HBA ports from /sys/class/fc_host), ``nvme_hostnqn`` (/etc/nvme/hostnqn) + and ``sdc_guid`` (PowerFlex SDC). Missing ones are left out. Read-only.""" + out: dict = {} + for line in _host_file("/etc/iscsi/initiatorname.iscsi").splitlines(): + line = line.strip() + if line.startswith("InitiatorName=") and not line.startswith("#"): + out["iscsi_iqn"] = line.split("=", 1)[1].strip() + break + wwpns = [] + base = "/sys/class/fc_host" + if os.path.isdir(base): + for host in sorted(os.listdir(base)): + name = _read_sysfs(f"{base}/{host}/port_name").lower() + if re.fullmatch(r"0x[0-9a-f]{16}", name) and int(name, 16): + wwpns.append(name.removeprefix("0x")) + if wwpns: + out["fc_wwpns"] = wwpns + nqn = _host_file("/etc/nvme/hostnqn").strip() + if nqn: + out["nvme_hostnqn"] = nqn + guid = "" + for drv_cfg in ("/opt/emc/scaleio/sdc/bin/drv_cfg", "/host/opt/emc/scaleio/sdc/bin/drv_cfg"): + if os.path.exists(drv_cfg): + guid, _, _ = shell_utils.run_command(f"{drv_cfg} --query_guid") + break + match = re.search(r"\b([0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12})\b", + guid or "") + if match: + out["sdc_guid"] = match.group(1) + return out + + +def scan_for_new_luns() -> dict: + """Make the kernel look for LUNs / namespaces newly presented to this host + by a storage array: every SCSI host (FC HBAs and iSCSI sessions alike) is + scanned (``- - -`` = all channels, targets, LUNs) and every NVMe + controller rescans its namespaces. Existing devices are not touched. + + The host must already have its SAN paths: iSCSI sessions logged in, FC + zoning done, NVMe-oF controllers connected. Returns ``{"scanned": [...]}``.""" + import glob as _glob + scanned: list[str] = [] + for path in sorted(_glob.glob("/sys/class/scsi_host/host*/scan")): + try: + _write_sysfs(path, "- - -") + scanned.append(path) + except OSError as e: + logger.warning("SCSI host scan %s failed: %s", path, e) + for path in sorted(_glob.glob("/sys/class/nvme/nvme*/rescan_controller")): + try: + _write_sysfs(path, "1") + scanned.append(path) + except OSError as e: + logger.warning("NVMe rescan %s failed: %s", path, e) + return {"scanned": scanned} def find_lun_devices(wwn: str) -> list[dict]: diff --git a/tests/unit/storage_arrays/test_san_plugin.py b/tests/unit/storage_arrays/test_san_plugin.py new file mode 100644 index 0000000000..e88bdfe14c --- /dev/null +++ b/tests/unit/storage_arrays/test_san_plugin.py @@ -0,0 +1,154 @@ +"""The SAN array plugins (simplyblock_core.array.san): every storage_arrays +adapter driven through the multi-attach ArrayPlugin contract, against its +vendor fake.""" +from __future__ import annotations + +import pytest + +from simplyblock_core import array, multi_attach +from simplyblock_core.array import san +from simplyblock_core.storage_arrays.base import Protocol +from simplyblock_core.storage_arrays.testing import FakeTransport + +from .conftest import VENDORS, fake_module + +GIB = 1 << 30 +LUN = "simplyblock-node-a-s0" + +_KEY = { + Protocol.ISCSI: "iscsi_iqn", + Protocol.FC: "fc_wwpns", + Protocol.NVME_TCP: "nvme_hostnqn", + Protocol.NVME_RDMA: "nvme_hostnqn", + Protocol.NVME_FC: "nvme_hostnqn", + Protocol.SDC: "sdc_guid", +} + + +class Bed: + def __init__(self, vendor: str): + self.vendor = vendor + self.setup = fake_module(vendor).contract_setup() + protos = list(self.setup.initiators) + self.protocol = Protocol.ISCSI if Protocol.ISCSI in protos else protos[0] + self.creds = {n: self.setup.credential.get(n) for n in self.setup.credential.field_names} + + def plugin(self, cluster_id: str = "cluster-1", **extra) -> san.SanArrayPlugin: + options = {**self.setup.config.options, "protocol": self.protocol.value, **extra} + p = array.plugin_class(self.vendor)(endpoint=self.setup.config.endpoint, options=options, + cluster_id=cluster_id) + assert isinstance(p, san.SanArrayPlugin) + p.transport = FakeTransport(self.setup.fake) + p.session_kwargs = {"sleep": lambda s: None} + p.connect(self.setup.config.endpoint, self.creds, options) + return p + + def initiators(self, which: int) -> dict: + group = self.setup.initiators[self.protocol][which] + key = _KEY[self.protocol] + addresses = [i.address for i in group] + value = addresses if key == "fc_wwpns" else addresses[0] + return {"hostname": f"node-{which}", key: value} + + +@pytest.fixture(params=VENDORS) +def bed(request) -> Bed: + return Bed(request.param) + + +def test_registered_as_builtin_plugins(): + assert set(san.SAN_PLUGINS) == set(VENDORS) + for name, path in san.SAN_PLUGINS.items(): + assert array.BUILTIN_PLUGINS[name] == path + assert array.plugin_class(name).vendor == name + + +def test_lifecycle_multi_attach_fence_grow_delete(bed): + p = bed.plugin() + caps = p.capabilities() + assert bed.protocol.value in caps["protocols"] + host_a = p.ensure_host("node-a", bed.initiators(0)) + assert p.ensure_host("node-a", bed.initiators(0)) == host_a # idempotent + + lun = p.create_lun(LUN, 10 * GIB, {"simplyblock-host": "node-a"}) + assert lun["size"] >= 10 * GIB + assert lun["wwn"] and not lun["wwn"].startswith("naa.") + assert p.create_lun(LUN, 10 * GIB, {})["id"] == lun["id"] # retried create + + p.attach(lun["id"], host_a) + p.attach(lun["id"], host_a) # idempotent + view = p.get(lun["id"]) + assert view["hosts"] == [host_a] + assert multi_attach.normalize_wwn(view["wwn"]) == multi_attach.normalize_wwn(lun["wwn"]) + + host_b = p.ensure_host("node-b", bed.initiators(1)) + p.attach(lun["id"], host_b) + assert sorted(p.get(lun["id"])["hosts"]) == sorted([host_a, host_b]) + p.detach(lun["id"], host_b) + assert p.get(lun["id"])["hosts"] == [host_a] + + assert p.fence(lun["id"], host_a) + assert p.get(lun["id"])["hosts"] == [] + assert p.unfence(lun["id"], host_a) + assert p.get(lun["id"])["hosts"] == [host_a] + + p.resize(lun["id"], 20 * GIB) + assert p.get(lun["id"])["size"] >= 20 * GIB + + p.delete_lun(lun["id"]) + with pytest.raises(array.ArrayError): + p.get(lun["id"]) + p.delete_lun(lun["id"]) # gone: succeeds + p.detach(lun["id"], host_a) # gone: succeeds + + +def test_other_clusters_luns_are_never_touched(bed): + mine = bed.plugin("cluster-1") + theirs = bed.plugin("cluster-2") + lun = mine.create_lun(LUN, 10 * GIB, {}) + # Same requested name, other cluster: a different LUN. + assert theirs.create_lun(LUN, 10 * GIB, {})["id"] != lun["id"] + for op in (lambda: theirs.get(lun["id"]), lambda: theirs.delete_lun(lun["id"]), + lambda: theirs.resize(lun["id"], 20 * GIB), + lambda: theirs.attach(lun["id"], "h"), lambda: theirs.detach(lun["id"], "h")): + with pytest.raises(array.ForeignLunError): + op() + assert mine.get(lun["id"])["size"] >= 10 * GIB + + foreign = bed.setup.create_foreign_volume("prod-db-01", 5 * GIB) + with pytest.raises(array.ForeignLunError): + mine.delete_lun(foreign) + + +def test_connect_validation(bed): + with pytest.raises(array.ArrayError, match="protocol"): + bed.plugin(protocol="") + with pytest.raises(array.ArrayError, match="unknown protocol"): + bed.plugin(protocol="carrier-pigeon") + unsupported = [p for p in Protocol if p.value not in + bed.plugin().capabilities()["protocols"]] + if unsupported: + with pytest.raises(array.ArrayError, match="does not support"): + bed.plugin(protocol=unsupported[0].value) + + +def test_host_without_initiator_of_the_protocol(bed): + p = bed.plugin() + with pytest.raises(array.ArrayError, match="reports no"): + p.ensure_host("node-a", {"hostname": "node-a"}) + + +def test_list_luns_falls_back_to_recorded(bed): + with pytest.raises(NotImplementedError): + bed.plugin().list_luns() + + +def test_names(): + assert len(san.lun_uuid("c", "x")) == 36 + a, b = san.lun_uuid("c1", LUN), san.lun_uuid("c2", LUN) + assert a != b and a == san.lun_uuid("c1", LUN) + assert san.host_object_name("node-a.example.com", 63) == "sb-node-a-example-com" + long = san.host_object_name("x" * 80, 31) + assert len(long) == 31 and long.startswith("sb-x") + assert san.host_object_name("x" * 80, 31) == long + assert san.host_object_name("y" * 80, 31) != long diff --git a/tests/unit/test_array_lifecycle.py b/tests/unit/test_array_lifecycle.py index 08a03fa341..3f115c6a6d 100644 --- a/tests/unit/test_array_lifecycle.py +++ b/tests/unit/test_array_lifecycle.py @@ -56,8 +56,10 @@ def clear(self, k): class FakeKMS: - """A KMS holding secrets in memory (as plaintext, like the stores).""" + """A KMS holding secrets in memory (as plaintext, like the stores), and + wrapping credentials by hex-encoding them (the real envelope format).""" store: ClassVar[dict] = {} + keks: ClassVar[set] = set() def __enter__(self): return self @@ -65,6 +67,23 @@ def __enter__(self): def __exit__(self, *a): return None + def ensure_key_encryption_key(self, name): + FakeKMS.keks.add(name) + + def _wrap_bytes(self, kek_name, plaintext): + return f"fake:{kek_name}:" + plaintext.hex() + + def _unwrap_bytes(self, kek_name, wrapped): + prefix = f"fake:{kek_name}:" + if not wrapped.startswith(prefix): + raise KMSException("wrong key") + return bytes.fromhex(wrapped[len(prefix):]) + + from simplyblock_core.kms._base import KMS as _KMS + wrap_secret = _KMS.wrap_secret + unwrap_secret = _KMS.unwrap_secret + del _KMS + def put_secret(self, path, data): from simplyblock_core.kms._base import secret_plaintext FakeKMS.store[path] = secret_plaintext(data) @@ -107,6 +126,8 @@ def _cluster(plugin=PLUGIN, multi=True, lun_size=0): cl.array_plugin = plugin cl.array_lun_size = lun_size cl.status = Cluster.STATUS_ACTIVE + # No external KMS in these tests: the local one, explicitly allowed. + cl.allow_local_kms_credentials = True return cl @@ -144,12 +165,14 @@ class _Case(unittest.TestCase): def setUp(self): FakeArrayPlugin.reset() FakeKMS.store = {} + FakeKMS.keks = set() array.register_plugin(PLUGIN, FakeArrayPlugin) self.addCleanup(array.unregister_plugin, PLUGIN) for target, attr, kw in ( (array_controller, "_event", {}), (array_controller, "_sleep", {}), - ("simplyblock_core.kms.create_kms_connection", None, {"side_effect": lambda c: FakeKMS()}), + ("simplyblock_core.kms.create_kms_connection", None, + {"side_effect": lambda c, **k: FakeKMS()}), ("simplyblock_core.models.lock.DbLock", None, {"side_effect": lambda *a, **k: nullcontext()})): p = patch(target, **kw) if attr is None else patch.object(target, attr, **kw) p.start() @@ -317,7 +340,7 @@ def test_foreign_lun_is_never_touched(self): class TestRegistration(_Case): - def test_register_stores_credentials_in_kms_only(self): + def test_register_keeps_only_wrapped_credentials(self): db = self.use_db(FakeDB(_cluster(plugin=""), [])) view = array_controller.register_array( CID, PLUGIN, "https://array", {"az": "z"}, {"password": "pw"}, lun_size=GIB) @@ -326,8 +349,14 @@ def test_register_stores_credentials_in_kms_only(self): self.assertEqual((cl.array_plugin, cl.array_endpoint, cl.array_options, cl.array_lun_size), (PLUGIN, "https://array", {"az": "z"}, GIB)) self.assertTrue(cl.array_has_credentials) - self.assertEqual(FakeKMS.store[array_secret_path(CID, cl.array_id)], {"password": "pw"}) + ref = cl.array_credential_ref + self.assertEqual((ref["kind"], ref["backend"], ref["kek_name"]), + ("managed", "local", f"credentials-{CID}")) + self.assertEqual(FakeKMS.store, {}) # nothing at the old KMS path self.assertNotIn("pw", json.dumps(cl.to_dict(unwrap_secrets=True), default=str)) + self.assertEqual(view["credential_ref"], {"kind": "managed", "backend": "local"}) + # Read back through the reference, bound to array id + endpoint. + self.assertEqual(array.array_credentials(cl)["password"].get_secret_value(), "pw") self.assertNotIn("credentials", view) self.assertNotIn("pw", json.dumps(view)) self.assertEqual(view["capabilities"]["max_attachments"], 16) @@ -342,8 +371,69 @@ def test_reregister_keeps_id_and_drops_old_secret(self): array_controller.register_array(CID, PLUGIN) self.assertEqual(db.cluster.array_id, aid) self.assertFalse(db.cluster.array_has_credentials) + self.assertEqual(db.cluster.array_credential_ref, {}) self.assertEqual(FakeKMS.store, {}) + def test_reregister_removes_a_legacy_kms_secret(self): + cl = _cluster() + cl.array_id, cl.array_has_credentials = "arr-1", True + FakeKMS.store[array_secret_path(CID, "arr-1")] = {"password": "old"} + db = self.use_db(FakeDB(cl, [])) + array_controller.register_array(CID, PLUGIN, credentials={"password": "new"}) + self.assertEqual(FakeKMS.store, {}) + self.assertEqual(db.cluster.array_credential_ref["kind"], "managed") + self.assertEqual(array.array_credentials(db.cluster)["password"].get_secret_value(), "new") + + def test_local_kms_needs_the_opt_in(self): + cl = _cluster(plugin="") + cl.allow_local_kms_credentials = False + db = self.use_db(FakeDB(cl, [])) + with self.assertRaisesRegex(ValueError, "allow_local_kms_credentials"): + array_controller.register_array(CID, PLUGIN, credentials={"password": "pw"}) + self.assertEqual((db.cluster.array_plugin, db.cluster.array_credential_ref), ("", {})) + # Without credentials (an instance role) nothing is stored: allowed. + array_controller.register_array(CID, PLUGIN) + self.assertEqual(db.cluster.array_plugin, PLUGIN) + + def test_endpoint_rewrite_does_not_release_the_credentials(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + array_controller.register_array(CID, PLUGIN, "https://array.example", + credentials={"password": "pw"}) + db.cluster.array_endpoint = "https://attacker.example" + with self.assertRaisesRegex(array.ArrayError, "different context"): + array.array_credentials(db.cluster) + + def test_external_vault_reference(self): + from simplyblock_core.kms import HCPClient + vault = MagicMock(spec=HCPClient) + vault.__enter__.return_value = vault + vault.read_kv_secret.return_value = {"password": SecretStr("ext")} + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + ref = {"kind": "vault-kv", "mount": "secret", "path": "arrays/a1"} + with patch("simplyblock_core.kms.create_kms_connection", side_effect=lambda c, **k: vault): + view = array_controller.register_array(CID, PLUGIN, credential_ref=ref) + self.assertEqual(FakeArrayPlugin.connected[-1][1]["password"].get_secret_value(), "ext") + self.assertEqual(array.array_credentials(db.cluster)["password"].get_secret_value(), "ext") + self.assertEqual(db.cluster.array_credential_ref["path"], "arrays/a1") + self.assertEqual(view["credential_ref"]["kind"], "vault-kv") + vault.read_kv_secret.assert_called_with("secret", "arrays/a1", None) + + def test_reference_refusals(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + with self.assertRaisesRegex(ValueError, "not both"): + array_controller.register_array(CID, PLUGIN, credentials={"p": "x"}, + credential_ref={"kind": "vault-kv", "mount": "m", "path": "p"}) + with self.assertRaisesRegex(ValueError, "external secret"): + array_controller.register_array(CID, PLUGIN, credential_ref={ + "kind": "managed", "backend": "local", "kek_name": "k", "wrapped": "w"}) + with self.assertRaisesRegex(ValueError, "not a valid reference"): + array_controller.register_array(CID, PLUGIN, credential_ref={"kind": "nope"}) + # A Vault reference on a cluster whose KMS is not Vault. + with self.assertRaisesRegex(ValueError, "could not be read"): + array_controller.register_array(CID, PLUGIN, credential_ref={ + "kind": "vault-kv", "mount": "m", "path": "p"}) + self.assertEqual(db.cluster.array_plugin, "") + def test_refusals_save_nothing(self): db = self.use_db(FakeDB(_cluster(plugin="", multi=False), [])) with self.assertRaisesRegex(ValueError, "multi-attach"): @@ -416,6 +506,37 @@ def test_provision_creates_attaches_and_waits(self): self.assertEqual(entry2["array_lun_id"], lun["id"]) self.assertEqual(len(FakeArrayPlugin.luns), 1) + def test_san_initiators_reach_the_array_and_the_host_scans(self): + cl = _cluster(lun_size=GIB) + api = self._api(appear_after=2) + api.scan_new_luns.return_value = ({"scanned": ["/sys/class/scsi_host/host0/scan"]}, None) + info = {"hostname": "worker-2", "system_id": "sys", + "san_initiators": {"iscsi_iqn": "iqn.2026-01.io.sb:w2", "fc_wwpns": ["2100"], + "unknown": "dropped"}} + _, host = array_controller.provision_node_lun(cl, api, info, {"socket": 1}, []) + inits = FakeArrayPlugin.hosts[host]["initiators"] + self.assertEqual((inits["iscsi_iqn"], inits["fc_wwpns"]), ("iqn.2026-01.io.sb:w2", ["2100"])) + self.assertNotIn("unknown", inits) + # One scan per inventory poll until the LUN showed up. + self.assertEqual(api.scan_new_luns.call_count, api.get_blockdevices.call_count) + self.assertGreaterEqual(api.scan_new_luns.call_count, 3) + + def test_agent_without_scan_endpoint_still_provisions(self): + api = self._api() + api.scan_new_luns.side_effect = RuntimeError("404") + entry, _ = array_controller.provision_node_lun( + _cluster(lun_size=GIB), api, {"hostname": "w"}, {"socket": 0}, []) + self.assertTrue(entry["array_lun_id"]) + + def test_existing_node_initiators_come_from_its_agent(self): + n = _node("a") + n.agent.info.return_value = ({"san_initiators": {"nvme_hostnqn": "nqn.h:a"}}, None) + plugin = self.plugin(_cluster()) + db = FakeDB(_cluster(), [n]) + host = array_controller.ensure_node_host(plugin, n, db) + self.assertEqual(FakeArrayPlugin.hosts[host]["initiators"]["nvme_hostnqn"], "nqn.h:a") + self.assertEqual(n.array_host_id, host) + def test_lun_that_never_appears_fails(self): cl = _cluster(lun_size=GIB) with self.assertRaisesRegex(array_controller.ArrayLifecycleError, "did not appear"): @@ -676,6 +797,7 @@ def test_release_never_detaches_a_lun_the_node_serves(self): def test_delete_cluster_deletes_only_its_luns_and_secret(self): cl = _cluster() + self.use_db(FakeDB(cl, [])) cl.array_id, cl.array_has_credentials = "arr", True FakeKMS.store[array_secret_path(CID, "arr")] = {"k": "v"} p = self.plugin(cl) @@ -686,8 +808,20 @@ def test_delete_cluster_deletes_only_its_luns_and_secret(self): self.assertEqual(sorted(FakeArrayPlugin.luns), ["lun-foreign"]) self.assertEqual(FakeKMS.store, {}) + def test_delete_cluster_uses_recorded_luns_when_listing_is_unsupported(self): + cl = _cluster() + a = _node("a") + self.use_db(FakeDB(cl, [a])) + p = self.plugin(cl) + lun = p.create_lun("a", GIB, {}) + _lun_device(a, lun["id"], lun["wwn"]) + with patch.object(FakeArrayPlugin, "list_luns", side_effect=NotImplementedError): + array_controller.delete_cluster_luns(cl) + self.assertEqual(FakeArrayPlugin.luns, {}) + def test_delete_cluster_failure(self): cl = _cluster() + self.use_db(FakeDB(cl, [])) p = self.plugin(cl) p.create_lun("a", GIB, {}) FakeArrayPlugin.fail.add("delete_lun") @@ -1008,7 +1142,7 @@ def test_set_array_reads_and_wraps_credentials(self): json.dump({"aws_access_key_id": "AK", "aws_secret_access_key": "SK"}, f) self.addCleanup(os.unlink, f.name) args = MagicMock(cluster_id=CID, plugin="aws-ebs", endpoint="", options='{"az": "z"}', - credentials_file=f.name, lun_size=GIB, json=True) + credentials_file=f.name, credential_ref=None, lun_size=GIB, json=True) with patch.object(array_controller, "register_array", return_value={"id": "arr"}) as reg: CLIWrapperBase.cluster__set_array(MagicMock(), "set-array", args) a = reg.call_args.args diff --git a/tests/unit/test_credentials.py b/tests/unit/test_credentials.py index 5761f9920e..2319232688 100644 --- a/tests/unit/test_credentials.py +++ b/tests/unit/test_credentials.py @@ -48,6 +48,10 @@ def ensure_key_encryption_key(self, name): if name not in self.keks: self.create_key_encryption_key(name) + def put_secret(self, path, data): ... + def get_secret(self, path): ... + def delete_secret(self, path): ... + def _wrap_bytes(self, kek_name, plaintext): if kek_name not in self.keks: raise KMSException("no such KEK") diff --git a/tests/unit/test_san_host_agent.py b/tests/unit/test_san_host_agent.py new file mode 100644 index 0000000000..f3371675bc --- /dev/null +++ b/tests/unit/test_san_host_agent.py @@ -0,0 +1,96 @@ +"""Node agent, SAN host side: the initiator report (how an array knows the +host) and the scan for newly presented LUNs.""" +import os +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from simplyblock_web import node_utils + + +class _Root(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + real_read, real_isdir, real_listdir = node_utils._read_sysfs, os.path.isdir, os.listdir + + def _map(p): + return os.path.join(self.root, p.lstrip("/")) if p.startswith("/") else p + for target, fn in ((node_utils, "_read_sysfs"),): + p = patch.object(target, fn, side_effect=lambda path: real_read(_map(path))) + p.start() + self.addCleanup(p.stop) + p = patch.object(node_utils.os.path, "isdir", side_effect=lambda path: real_isdir(_map(path))) + p.start() + self.addCleanup(p.stop) + p = patch.object(node_utils.os, "listdir", side_effect=lambda path: real_listdir(_map(path))) + p.start() + self.addCleanup(p.stop) + self.map = _map + + def mk(self, rel, content=""): + path = os.path.join(self.root, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as fh: + fh.write(content) + + +class TestInitiators(_Root): + + def test_reports_iqn_wwpns_and_hostnqn(self): + self.mk("etc/iscsi/initiatorname.iscsi", + "## generated\n#InitiatorName=iqn.ignored\nInitiatorName=iqn.1994-05.com.redhat:abc\n") + self.mk("sys/class/fc_host/host3/port_name", "0x21000024FF4C1A2B\n") + self.mk("sys/class/fc_host/host4/port_name", "0x0000000000000000\n") # unconfigured port + self.mk("etc/nvme/hostnqn", "nqn.2014-08.org.nvmexpress:uuid:1234\n") + with patch.object(node_utils.os.path, "exists", return_value=False): + out = node_utils.get_san_initiators() + self.assertEqual(out, { + "iscsi_iqn": "iqn.1994-05.com.redhat:abc", + "fc_wwpns": ["21000024ff4c1a2b"], + "nvme_hostnqn": "nqn.2014-08.org.nvmexpress:uuid:1234", + }) + + def test_kubernetes_agent_reads_the_hosts_etc(self): + self.mk("host/etc/nvme/hostnqn", "nqn.host") + self.mk("etc/nvme/hostnqn", "nqn.container") + with patch.object(node_utils.os.path, "exists", return_value=False): + self.assertEqual(node_utils.get_san_initiators()["nvme_hostnqn"], "nqn.host") + + def test_powerflex_sdc_guid(self): + with patch.object(node_utils.os.path, "exists", + side_effect=lambda p: p == "/opt/emc/scaleio/sdc/bin/drv_cfg"), \ + patch.object(node_utils.shell_utils, "run_command", + return_value=("INFO: DRV_CFG: Query GUID: 0E9B4C8A-1234-4ABC-9DEF-00112233AABB", + "", 0)): + self.assertEqual(node_utils.get_san_initiators()["sdc_guid"], + "0E9B4C8A-1234-4ABC-9DEF-00112233AABB") + + def test_nothing_configured(self): + with patch.object(node_utils.os.path, "exists", return_value=False): + self.assertEqual(node_utils.get_san_initiators(), {}) + + +class TestScan(_Root): + + def test_scans_scsi_hosts_and_nvme_controllers(self): + written = {} + paths = ["/sys/class/scsi_host/host0/scan", "/sys/class/scsi_host/host7/scan", + "/sys/class/nvme/nvme2/rescan_controller"] + with patch("glob.glob", side_effect=lambda pat: [p for p in paths if p.startswith(pat.split("*")[0])]), \ + patch.object(node_utils, "_write_sysfs", side_effect=lambda p, v: written.__setitem__(p, v)): + out = node_utils.scan_for_new_luns() + self.assertEqual(out["scanned"], paths) + self.assertEqual(written["/sys/class/scsi_host/host7/scan"], "- - -") + self.assertEqual(written["/sys/class/nvme/nvme2/rescan_controller"], "1") + + def test_a_failing_host_does_not_stop_the_scan(self): + paths = ["/sys/class/scsi_host/host0/scan", "/sys/class/scsi_host/host1/scan"] + + def write(p, v): + if p.endswith("host0/scan"): + raise OSError("busy") + with patch("glob.glob", side_effect=lambda pat: [p for p in paths if "scsi_host" in pat]), \ + patch.object(node_utils, "_write_sysfs", side_effect=write): + self.assertEqual(node_utils.scan_for_new_luns()["scanned"], paths[1:]) From cc8d21cdc0fad96623c318c522fac65384448e4d Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 16:45:19 +0100 Subject: [PATCH 31/50] fix(storage_arrays): adapters checked against vendor source No array or simulator was available, so every adapter's "Verify against a real system" list was checked against the vendors' published clients (SDKs, CSI drivers, Ansible collections, Cinder drivers). Confirmed items moved to a "Checked against vendor source" list in each docstring; contradicted ones fixed in adapter, fake and tests: - NetApp: LUN/namespace calls may answer 202 with a job; igroup protocol cannot be changed (conflict instead of PATCH); errors classified by ONTAP code; ASA r2 refused; option skip_recovery_queue (FlexVol delete force). - Pure: NVMe connections report nsid, not lun. - PowerStore: duplicate name may be 500; detach of a detached volume is "missing"; NVMe nsid read from the volume. - PowerMax: NVMe initiator ids are :; vendor not-found wording; unknown job states fail at once; workloadSelection "None". - PowerFlex: /api/login on 4.x too, basic vs Bearer chosen by /api/version; allowMultipleMappings "true". - HPE WSAPI: code 29 is not a conflict; non-auth 403s (120/150/153/215) no longer trigger re-login; personas and reduce documented correctly. - HPE Nimble: 202 async jobs polled (job_timeout_s); already-offline volume tolerated on delete. - IBM: port-taken codes CMMVC6578E/9328E; mkhost with one port kind, the rest via addhostport. - Hitachi: resource-group lock under /services; LDEV lookup pages the whole range (missed LDEVs above 16383); structured KART/SSB error codes. - Huawei: literal NAME:: filter; host initiators re-checked by PARENTID before unbinding. - Infinidat: approved=true on every mutation; APPROVAL_REQUIRED* codes; /system model field. infinisim is not public (README). Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/storage_arrays/README.md | 21 +- .../storage_arrays/adapters/dell_powerflex.py | 129 +++++++---- .../storage_arrays/adapters/dell_powermax.py | 123 +++++++--- .../adapters/dell_powerstore.py | 103 ++++++--- .../storage_arrays/adapters/hitachi_vsp.py | 171 +++++++++----- .../storage_arrays/adapters/hpe_nimble.py | 146 +++++++++--- .../storage_arrays/adapters/hpe_wsapi.py | 93 +++++--- .../storage_arrays/adapters/huawei_dorado.py | 78 +++++-- .../adapters/ibm_flashsystem.py | 125 +++++++---- .../storage_arrays/adapters/infinidat.py | 99 +++++--- .../storage_arrays/adapters/netapp_ontap.py | 151 ++++++++++--- .../storage_arrays/adapters/pure.py | 55 ++++- .../storage_arrays/testing/dell_powerflex.py | 38 ++-- .../storage_arrays/testing/dell_powermax.py | 39 ++-- .../storage_arrays/testing/dell_powerstore.py | 85 +++++-- .../storage_arrays/testing/hitachi_vsp.py | 63 ++++-- .../storage_arrays/testing/hpe_nimble.py | 71 +++++- .../storage_arrays/testing/hpe_wsapi.py | 54 +++-- .../storage_arrays/testing/huawei_dorado.py | 8 +- .../storage_arrays/testing/ibm_flashsystem.py | 17 +- .../storage_arrays/testing/infinidat.py | 34 ++- .../storage_arrays/testing/netapp_ontap.py | 212 ++++++++++++------ .../storage_arrays/testing/pure.py | 24 +- .../storage_arrays/test_dell_powerflex.py | 45 +++- .../unit/storage_arrays/test_dell_powermax.py | 75 +++++++ .../storage_arrays/test_dell_powerstore.py | 36 +++ tests/unit/storage_arrays/test_hitachi_vsp.py | 67 +++++- tests/unit/storage_arrays/test_hpe_nimble.py | 59 +++++ tests/unit/storage_arrays/test_hpe_wsapi.py | 57 ++++- .../unit/storage_arrays/test_huawei_dorado.py | 23 ++ .../storage_arrays/test_ibm_flashsystem.py | 29 +++ tests/unit/storage_arrays/test_infinidat.py | 34 +++ .../unit/storage_arrays/test_netapp_ontap.py | 108 ++++++++- tests/unit/storage_arrays/test_pure.py | 130 +++++++++++ 34 files changed, 2050 insertions(+), 552 deletions(-) create mode 100644 tests/unit/storage_arrays/test_pure.py diff --git a/simplyblock_core/storage_arrays/README.md b/simplyblock_core/storage_arrays/README.md index 83bbbd7696..10aa886214 100644 --- a/simplyblock_core/storage_arrays/README.md +++ b/simplyblock_core/storage_arrays/README.md @@ -22,6 +22,25 @@ There is no usable cross-vendor standard, so each array has its own adapter. SMI Each adapter's module docstring lists the options it reads, the credential fields it needs, and a section called **"Verify against a real system"**. That section names every API detail its author was unsure of. Work through it before the adapter is first used against a real array. +## Using an array with a cluster + +The adapters are the array plugins of the multi-attach array LUN mode (`simplyblock_core.array.san`, one plugin per key above). +Register the array on a multi-attach cluster: + +``` +sbctl cluster set-array --plugin pure-flasharray --endpoint https://pure01.example \ + --options '{"protocol": "iscsi", "ca_bundle": "-----BEGIN CERTIFICATE-----..."}' \ + --credentials-file pure01.json --lun-size 4T +``` + +- `protocol` (required): `iscsi`, `fc`, `nvme-tcp`, `nvme-rdma`, `nvme-fc` or `sdc`. The nodes are registered on the array with the initiators of that protocol, as their agents report them. +- `ca_bundle`, `verify_tls`, `timeout_s`, plus the adapter's own options (pool, SVM, CPG, port group, ...; see the adapter docstring). +- Credentials: `--credentials-file` (wrapped by the cluster's KMS, bound to the array id and endpoint; the local KMS only with `allow_local_kms_credentials`), or `--credential-ref` naming a Vault KV / KMIP secret simplyblock only reads. + +Each node added afterwards gets a LUN of `--lun-size` created, mapped to its host, and found after a SCSI/NVMe rescan; the LUN is presented to the node's secondary and tertiary as well. +The hosts must already have their SAN paths: iSCSI sessions logged in, FC zoning done, NVMe-oF controllers connected. +LUNs are named `sb-`; a cluster never acts on a LUN of another name. + ## Layers - `base.py`: the adapter contract (`ArrayAdapter`) and the shared model. @@ -62,7 +81,7 @@ These are partly from memory; check availability with each vendor. - **HPE:** the 3PAR StoreServ simulator (older WSAPI). Alletra through HPE demo and partner labs. - **IBM:** Storage Virtualize for Public Cloud, plus IBM Technology Zone labs. - **Hitachi, Huawei:** partner and demo labs only. -- **Infinidat:** `infinisim`, an open-source Python simulator of the InfiniBox API. +- **Infinidat:** partner and demo labs. (`infinisim`, the simulator infinisdk's own tests use, is not published.) Vendor SDKs make good reference code for API details: `py-pure-client`, `netapp-ontap`, `PyPowerStore`, `PyU4V`, `PyPowerFlex`, `hpe3parclient`, `nimble-sdk`, `infinisdk`. The same goes for each vendor's Ansible collection, CSI driver and OpenStack Cinder driver. diff --git a/simplyblock_core/storage_arrays/adapters/dell_powerflex.py b/simplyblock_core/storage_arrays/adapters/dell_powerflex.py index 12e017bb2f..93bbf4806c 100644 --- a/simplyblock_core/storage_arrays/adapters/dell_powerflex.py +++ b/simplyblock_core/storage_arrays/adapters/dell_powerflex.py @@ -15,8 +15,12 @@ ``POST /rest/auth/logout {"refresh_token"}`` ends the session. The access token is short-lived (minutes); rather than refreshing it, the adapter logs in again on the 401 (``ArrayClient`` does this transparently). -* ``auto`` (default): try ``oauth``; a 404/405 from ``/rest/auth/login`` - means a 3.x gateway, so fall back to ``token``. +* ``auto`` (default): ``GET /api/login`` as for ``token`` (served by 3.x + and 4.x gateways alike), then ``GET /api/version`` (a JSON string such as + ``"3.6"`` / ``"4.5"``): below 4.0 the token goes in basic auth as for + ``token``, from 4.0 on as ``Authorization: Bearer ``. This is how + Dell's own clients pick the scheme (goscaleio ``Authenticate`` / + ``BearerAuthenticationMinVersion``; PyPowerFlex ``get_api_version``). Volumes are addressed by their 16-hex-digit id (``Volume::``). Sizes are in KiB on create and GiB on resize and are rounded up by the array to a @@ -47,41 +51,77 @@ Mappings are SDC mappings (``addMappedSdc`` / ``removeMappedSdc``); the volume's ``mappedSdcInfo`` lists them (``null``, not ``[]``, when there are -none). Mapping a volume to a second SDC needs ``allowMultipleMappings``. +none). Mapping a volume to a second SDC needs ``allowMultipleMappings`` +(sent as ``"true"``, as csi-powerflex does). SDC mappings have no LUN number (``Mapping.lun`` is ``None``) and SDC volumes no NAA WWN or NGUID, so ``Volume.identifier`` is ``id.``. NVMe/TCP (4.x NVMe hosts, ``/api/types/Host/instances``) is not -implemented: a 4.x NVMe host has exactly one NQN, and the mapping actions -for NVMe hosts could not be verified against documentation (see below). +implemented: a 4.x NVMe host has exactly one NQN and is mapped with +``addMappedHost`` rather than ``addMappedSdc`` (see below). Errors: the gateway answers ``{"message", "httpStatusCode", "errorCode"}``, and reports "not found" as HTTP **500** (sometimes 400) rather than 404, so missing objects are recognised by message text; any other 500 stays an :class:`ArrayRequestError`. Booleans and numbers in action bodies are sent as -strings (``"TRUE"``, ``"8388608"``), as the REST reference shows them. - -Verify against a real system: - -* ``auto`` detection: that a 3.x gateway answers ``POST /rest/auth/login`` - with 404/405 (not 401), and whether a 4.x gateway still serves - ``GET /api/login``. +strings (``"true"``, ``"8388608"``), as Dell's clients send them. + +Checked against vendor source +----------------------------- +(PyPowerFlex = github.com/dell/python-powerflex, goscaleio = +github.com/dell/goscaleio, csi-powerflex = github.com/dell/csi-powerflex, +ansible-powerflex = github.com/dell/ansible-powerflex) + +* ``GET /api/login`` (basic user:password -> JSON string token), basic + ``user:`` on 3.x, ``GET /api/logout`` (PyPowerFlex + ``base_client.py`` ``_login`` / ``_logout``); a 4.x gateway still serves + ``/api/login`` and takes that token as a Bearer token (goscaleio + ``api.go`` ``basicAuthenticate``, ``api/api.go``); ``GET /api/version`` + picks the scheme (PyPowerFlex ``is_version_3``). +* 4.x OAuth: ``POST /rest/auth/login {"username", "password"}`` -> + ``{"access_token", "refresh_token"}``, ``POST /rest/auth/logout + {"refresh_token"}`` answering 204 (PyPowerFlex ``_appliance_login`` / + ``_appliance_logout``). +* Error texts: "Could not find the volume" (csi-powerflex + ``sioGatewayVolumeNotFound``); ``queryIdByKey`` for a missing name answers + "Not found" (goscaleio ``instance.go`` ``GetVolume``); "Volume name + already in use. Please use a different name." + (``sioGatewayVolumeNameInUse``). +* Gen1 arrays create volumes in multiples of 8 GiB, rounding up + (csi-powerflex ``validateVolSize``); Gen2/EC arrays use 1 GiB, which the + 8 GiB granularity also satisfies. +* ``addMappedSdc {"sdcId", "allowMultipleMappings", "accessMode"}`` with + ``accessMode`` ``ReadWrite`` (goscaleio ``MapVolumeSdcParam``; PyPowerFlex + ``add_mapped_sdc``; ansible-powerflex ``get_access_mode``); + ``allowMultipleMappings`` as a boolean string (csi-powerflex + ``strconv.FormatBool``, PyPowerFlex ``str(bool)``). +* ``removeMappedSdc``, ``setVolumeSize {"sizeInGB"}``, ``removeVolume + {"removeMode": "ONLY_ME"}``, ``queryIdByKey {"name"}``, create body + ``volumeSizeInKb`` / ``storagePoolId`` / ``volumeType`` as strings + (goscaleio ``sdc.go``, ``volume.go``, ``instance.go``, + ``types/v1/types.go`` ``VolumeParam``). +* Fields ``Sdc.sdcGuid`` / ``Sdc.name``, ``Volume.mappedSdcInfo[].sdcId``, + ``System.systemVersionName`` (goscaleio ``types/v1/types.go``). +* NVMe hosts map with ``addMappedHost {"hostId", ...}`` (goscaleio + ``nvme_host.go`` ``MapVolumeNVMe``) -- not implemented here. + +Verify against a real system +---------------------------- +* ``auth=oauth`` against a 3.x gateway (assumed: 404 for + ``/rest/auth/login``); whether ``GET /api/logout`` ends a Bearer session + on 4.x (the adapter calls it best-effort). * 4.x token lifetime and whether a refresh-token grant would be preferable - to re-login; the 4.x logout body. -* Exact error texts/codes: volume not found (``"Could not find the - volume"``, errorCode 79?), ``queryIdByKey`` for a missing name, name in - use, "already mapped", "not mapped", SDC name in use. + to re-login. +* Status codes (500 vs 400 vs 404) and the wording for "already mapped", + "not mapped" and an SDC name in use. * That ``setVolumeSize`` rounds a non-multiple of 8 up rather than rejecting it (the adapter always sends multiples of 8, so this only matters for volumes created outside simplyblock). -* ``accessMode: ReadWrite`` accepted by ``addMappedSdc`` on 3.6 and 4.x; - ``allowMultipleMappings`` spelling/value ``"TRUE"``. -* Field names ``sdcGuid`` and ``name`` on ``Sdc`` and ``mappedSdcInfo[].sdcId`` - on ``Volume``; ``systemVersionName`` on ``System``. * Whether SDC names are unique system-wide (the fake assumes so). """ from __future__ import annotations +import re from typing import Any from ..base import ( @@ -112,6 +152,11 @@ _NOT_MAPPED = ("not mapped",) +def _json_string(text: str | None) -> str: + """A body that is a bare JSON string (``"abc"``), unquoted.""" + return (text or "").strip().strip('"') + + def _message(body: Any, fallback: str) -> str: if isinstance(body, dict) and body.get("message"): msg = str(body["message"]) @@ -135,8 +180,9 @@ class DellPowerFlexAdapter(ArrayAdapter): required=True), OptionDoc("protection_domain_id", "Protection domain of the storage pool " "(sent on create; only needed where the gateway asks for it)."), - OptionDoc("auth", "'token' (3.x /api/login), 'oauth' (4.x /rest/auth/login) " - "or 'auto' (try oauth, fall back to token).", default="auto"), + OptionDoc("auth", "'token' (3.x /api/login, basic auth), 'oauth' (4.x " + "/rest/auth/login) or 'auto' (/api/login, then basic or Bearer by " + "/api/version).", default="auto"), OptionDoc("sdc_naming", "'claim': name unnamed SDCs after the host and treat an " "SDC named differently as another host's; 'ignore': never read or " "write SDC names.", default="claim"), @@ -164,30 +210,37 @@ def login(self) -> None: self._reset_auth() mode = self.option("auth") user, password = self.secret("username"), self.secret("password") - if mode in ("auto", "oauth"): + if mode == "oauth": resp = self.session.post("/rest/auth/login", - json={"username": user, "password": password}, - ok=(404, 405) if mode == "auto" else ()) - if resp.status_code < 300: - body = json_of(resp) or {} - token = body.get("access_token") if isinstance(body, dict) else None - if not token: - raise ArrayRequestError("PowerFlex /rest/auth/login returned no access_token") - self.session.headers["Authorization"] = f"Bearer {token}" - self._refresh_token = str(body.get("refresh_token") or "") - self._mode = "oauth" - return - # 404/405 under "auto": a 3.x gateway without PowerFlex Manager. + json={"username": user, "password": password}) + body = json_of(resp) or {} + token = body.get("access_token") if isinstance(body, dict) else None + if not token: + raise ArrayRequestError("PowerFlex /rest/auth/login returned no access_token") + self.session.headers["Authorization"] = f"Bearer {token}" + self._refresh_token = str(body.get("refresh_token") or "") + self._mode = "oauth" + return resp = self.session.get("/api/login", auth=(user, password)) - token = (resp.text or "").strip().strip('"') + token = _json_string(resp.text) if not token: raise ArrayRequestError("PowerFlex /api/login returned no token") + if mode == "auto": + version = _json_string(self.session.get("/api/version", auth=(user, token)).text) + m = re.match(r"^(\d+)\.", version) + if not m: + raise ArrayRequestError(f"PowerFlex /api/version returned {version!r}") + if int(m.group(1)) >= 4: + # goscaleio: 4.x takes the /api/login token as a Bearer token. + self.session.headers["Authorization"] = f"Bearer {token}" + self._mode = "bearer" + return self.session.set_basic_auth(user, token) self._mode = "token" def close(self) -> None: try: - if self._mode == "token": + if self._mode in ("token", "bearer"): self.session.get("/api/logout") elif self._mode == "oauth": self.session.post("/rest/auth/logout", @@ -335,7 +388,7 @@ def map_volume(self, volume_id: str, host_id: str) -> Mapping: if m.host_id == host_id: return m self._action(volume_id, "addMappedSdc", { - "sdcId": host_id, "allowMultipleMappings": "TRUE", "accessMode": "ReadWrite"}) + "sdcId": host_id, "allowMultipleMappings": "true", "accessMode": "ReadWrite"}) return Mapping(volume_id=volume_id, host_id=host_id, lun=None) def unmap_volume(self, volume_id: str, host_id: str) -> None: diff --git a/simplyblock_core/storage_arrays/adapters/dell_powermax.py b/simplyblock_core/storage_arrays/adapters/dell_powermax.py index a05fbbd1fd..19de6dfce0 100644 --- a/simplyblock_core/storage_arrays/adapters/dell_powermax.py +++ b/simplyblock_core/storage_arrays/adapters/dell_powermax.py @@ -3,7 +3,8 @@ Auth: HTTP basic on every request (no session). Provisioning lives under ``/univmax/restapi/{api_version}/sloprovisioning/symmetrix/{symmetrix_id}``. Errors are ``{"message": "..."}``, mostly with status 400; 404 for missing -objects; 401 for a rejected credential. +objects, 409 for "already exists", some 500s; 401 for a rejected credential. +They are classified by wording (see below). Layout (one object set per simplyblock volume): @@ -34,35 +35,66 @@ ``202 Accepted``; otherwise it sends ``SYNCHRONOUS`` but still follows a 202 should the server answer with a job. +Checked against vendor source +----------------------------- +(PyU4V = github.com/dell/PyU4V, gopowermax = github.com/dell/gopowermax, +csi-powermax = github.com/dell/csi-powermax) + +* ``sloId: "None"`` for no service level, with ``workloadSelection: "None"`` + (PyU4V ``provisioning.py`` ``create_storage_group``; gopowermax + ``GetCreateStorageGroupPayload``). The adapter sends both. +* An empty ``GET .../volume?volume_identifier=x`` answers ``count: 0`` + without ``resultList`` (PyU4V ``get_volume_list`` reads ``resultList`` + only when ``count > 0``); the filter is an exact match, ```` is the + substring form (gopowermax ``GetVolumeIDsIterator``); ``storageGroupId`` + filter on the same endpoint (``GetVolumesInStorageGroupIterator``). +* Error body ``{"message"}`` (gopowermax ``types.Error``). Missing objects + read "Could not find volume ...", "... cannot be found" (csi-powermax + ``controller.go`` ``cannotBeFound`` / ``errorNotFound``), removing a + device its storage group lacks "does not contain" (``deletion_worker.go``); + all three are classified as missing. "already exists" is a conflict. +* Delete order: remove the device from its storage group, then + ``DELETE volume`` -- no deallocation step (csi-powermax deletion worker + states ``disAssociateSG`` -> ``deleteVolume``; PyU4V ``delete_volume``). +* iSCSI and NVMe hosts are created before the node logs in to the array + (csi-powermax ``setupArrayForIscsi`` / ``setupArrayForNVMeTCP``). +* NVMe host initiator ids are ``:`` with the host ID as + 32 upper-case hex digits, not the bare NQN (csi-powermax + ``makeNVMeInitiatorIDs``). The adapter accepts an address already in that + form, or derives the host ID from a ``nqn.2014-08.org.nvmexpress:uuid:`` + NQN; other NQNs are refused (:class:`ArrayUnsupportedError`). +* Volume fields ``nguid``, ``effective_wwn``, ``storageGroupId``, + ``cap_cyl`` (gopowermax ``types/v100/volume.go``); 1 CYL = 1 966 080 + bytes (csi-powermax ``cylinderSizeInBytes``). +* ``maskingview/{mv}/connections?volume_id=`` rows carry + ``host_lun_address`` in hex (csi-powermax ``node.go`` + ``strconv.ParseInt(..., 16, 0)``); a view may have no rows yet + (csi-powermax vSphere path), giving ``lun=None``. +* Jobs: ``SUCCEEDED`` is done; ``CREATED``/``SCHEDULED``/``RUNNING``/ + ``VALIDATING``/``VALIDATED`` are in progress; any other state is a failure + whose text is in ``result`` (PyU4V ``common.py`` ``_is_job_finished``, + ``constants.INCOMPLETE_LIST``). + Verify against a real system ---------------------------- -* Whether ``sloId: "None"`` (no service level) is accepted on 10.x, and that - ``workloadSelection`` may be omitted. -* The empty answer of ``GET .../volume?volume_identifier=x`` (assumed: - ``count: 0`` with no ``resultList``) and whether the filter is exact or - case-insensitive; iterator paging (``/common/Iterator``) is not followed -- - fine for name lookups that match one device. -* The ``storageGroupId`` filter on ``GET .../volume``. -* Status codes and wording for: SG / host / masking view name already in use, - initiator already in another initiator group (classified by "already" / - "in use" / "exist" / "member of"), removing the last device of a masked SG, - deleting a device still in an SG, shrink. -* Whether a device must be removed from its SG before ``DELETE volume`` (done - here) and whether 10.x still needs a deallocation step first (not done). -* Whether iSCSI IQNs / NVMe host NQNs must already be logged in to the array - before ``POST host`` accepts them (FC WWNs need not be). -* NVMe/TCP: initiator ids for NVMe hosts (assumed: the host NQN) and whether - the volume object carries an ``nguid`` field (read if present). -* ``GET maskingview/{mv}/connections?volume_id=`` returns rows only for - initiator/port pairs (assumed: one row per initiator x port, logged in or - not); ``host_lun_address`` is hex. With no rows the mapping's LUN is None. -* Job states (``SUCCEEDED`` / ``FAILED`` / ``VALIDATE_FAILED``; others are - in progress) and that the failure text is in ``result``. +* Status codes: whether "already exists" comes as 409 or 400 and "cannot be + found" on masking view create as 404 or 500 (both handled), and the + wording for a shrink, removing the last device of a masked SG, deleting a + device still in an SG and an initiator in another host. +* Whether FC WWNs must be visible to the array (logged in / zoned) before + ``POST host`` accepts them; csi-powermax only registers WWNs the array + already lists in ``GET .../initiator``. +* That ``/etc/nvme/hostid`` equals the UUID in a uuid-form host NQN (the + nvme-cli default) on the hosts in use; otherwise pass ``:``. +* Whether ``volume_identifier`` matching is case-sensitive; iterator paging + (``/common/Iterator``) is not followed -- fine for lookups by name or + storage group that match one device. """ from __future__ import annotations import hashlib import math +import re from collections.abc import Callable from typing import Any @@ -88,10 +120,12 @@ CYLINDER_BYTES = 1920 * 1024 MAX_NAME = 64 _JOB_DONE = "SUCCEEDED" -_JOB_FAILED = frozenset({"FAILED", "VALIDATE_FAILED"}) +# PyU4V constants.INCOMPLETE_LIST; any other non-SUCCEEDED state is terminal. +_JOB_RUNNING = frozenset({"CREATED", "SCHEDULED", "RUNNING", "VALIDATING", "VALIDATED"}) _JOB_POLL_S = 2.0 -_NOT_FOUND = ("cannot find", "not found", "does not exist", "no such") +_NOT_FOUND = ("cannot find", "could not find", "cannot be found", "not found", + "does not exist", "does not contain", "no such") _CONFLICT = ("already", "in use", "exists", "member of", "is part of", "masking view") @@ -121,6 +155,28 @@ def powermax_wwn(item: dict[str, Any]) -> str | None: return f"naa.{v}" if v else None +_HOST_ID_RE = re.compile(r"^(?Pnqn\..+):(?P[0-9A-Fa-f]{32})$") +_UUID_NQN_RE = re.compile( + r"^nqn\.2014-08\.org\.nvmexpress:uuid:(?P[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-" + r"[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12})$") + + +def powermax_nvme_initiator(address: str) -> str: + """PowerMax NVMe initiator id: ``:`` (csi-powermax + ``makeNVMeInitiatorIDs``), HOSTID being the 32 upper-case hex digits of + ``/etc/nvme/hostid``. Accepts that form, or a uuid-form NQN whose UUID + is taken as the host ID.""" + m = _HOST_ID_RE.match(address) + if m: + return f"{m.group('nqn')}:{m.group('hostid').upper()}" + m = _UUID_NQN_RE.match(address) + if m: + return f"{address}:{m.group('uuid').replace('-', '').upper()}" + raise ArrayUnsupportedError( + f"PowerMax needs the NVMe host ID with the host NQN: give {address!r} as " + "':<32-hex host ID from /etc/nvme/hostid>'") + + def initiator_protocol(initiator_id: str) -> Protocol: """Protocol of an initiator-group member as Unisphere lists it.""" v = initiator_id.lower() @@ -178,8 +234,11 @@ def _exec(self) -> str: def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, json: Any = None, missing_ok: bool = False) -> Any: - resp = self.session.request(method, path, params=params, json=json, ok=(400, 404)) - if resp.status_code in (400, 404): + # 409: "already exists"; 500: e.g. "Port Group on Symmetrix cannot be + # found" on masking view create. Both are classified by wording. + resp = self.session.request(method, path, params=params, json=json, + ok=(400, 404, 409, 500)) + if resp.status_code in (400, 404, 409, 500): err = _classify(f"{method} {path} -> HTTP {resp.status_code}", _message(resp), resp.status_code) if missing_ok and isinstance(err, ArrayNotFoundError): @@ -197,8 +256,8 @@ def _wait_job(self, job_id: str, what: str) -> dict[str, Any]: status = str(job.get("status", "")).upper() if status == _JOB_DONE: return job - if status in _JOB_FAILED: - raise _classify(f"{what} (job {job_id} {status})", + if status not in _JOB_RUNNING: + raise _classify(f"{what} (job {job_id} {status or 'no status'})", str(job.get("result") or "no result")) self._sleep(_JOB_POLL_S) raise ArrayRequestError(f"{what}: job {job_id} did not finish within " @@ -259,6 +318,7 @@ def create_volume(self, name: str, size_bytes: int) -> Volume: "emulation": "FBA", "sloBasedStorageGroupParam": [{ "sloId": self.option("service_level") or "None", + "workloadSelection": "None", "volumeAttributes": [attrs], }], "executionOption": self._exec(), @@ -310,7 +370,7 @@ def delete_volume(self, volume_id: str) -> None: for sg in item.get("storageGroupId") or []: self._call("PUT", f"{self._slo}/storagegroup/{sg}", json={ "editStorageGroupActionParam": {"removeVolumeParam": {"volumeId": [volume_id]}}, - "executionOption": self._exec()}) + "executionOption": self._exec()}, missing_ok=True) if sg == name: self._call("DELETE", f"{self._slo}/storagegroup/{sg}", missing_ok=True) self._call("DELETE", f"{self._slo}/volume/{volume_id}") @@ -318,7 +378,8 @@ def delete_volume(self, volume_id: str) -> None: # -- hosts ------------------------------------------------------------------ def ensure_host(self, spec: HostSpec) -> str: - wanted = [i.normalized() for i in spec.initiators] + wanted = [powermax_nvme_initiator(i.address) if i.protocol.is_nvme else i.normalized() + for i in spec.initiators] host = self._call("GET", f"{self._slo}/host/{spec.name}", missing_ok=True) if host is None: self._call("POST", f"{self._slo}/host", json={ diff --git a/simplyblock_core/storage_arrays/adapters/dell_powerstore.py b/simplyblock_core/storage_arrays/adapters/dell_powerstore.py index 4a480d92e4..c85e3eacf7 100644 --- a/simplyblock_core/storage_arrays/adapters/dell_powerstore.py +++ b/simplyblock_core/storage_arrays/adapters/dell_powerstore.py @@ -5,44 +5,76 @@ ``DELL-EMC-TOKEN`` header; every later request sends the cookie, and every non-GET request must also send the token header (the array answers 403 without it). Basic auth is *not* kept on the session afterwards, so an expired -session surfaces as 401 and :class:`ArrayClient` logs in again. +session surfaces as 401 or 403 and :class:`ArrayClient` logs in again. -Queries use the PostgREST-style syntax: ``select=a,b`` picks fields, -``name=eq.x`` filters, ``limit``/``offset`` page; a partial page is answered -with ``206 Partial Content`` and a ``Content-Range`` header. +Queries use the PostgREST-style syntax: ``select=a,b`` picks fields (with +embedded objects as ``host(host_initiators)``), ``name=eq.x`` filters, +``limit``/``offset`` page; a partial page is answered with +``206 Partial Content`` and a ``Content-Range`` header. Errors: ``{"messages": [{"code": "0x...", "severity": "Error", -"message_l10n": "...", "arguments": [...]}]}``. Semantic failures (name in -use, initiator owned by another host, volume still attached, shrink) come -back as ``422 Unprocessable Entity``; malformed requests as 400. +"message_l10n": "...", "arguments": [...]}]}``. Semantic failures (initiator +owned by another host, volume still attached, shrink) come back as +``422 Unprocessable Entity``; a duplicate volume name as 422 or 500; malformed +requests, a missing host and a detach of a volume that is not attached +(``"Host is not attached to volume"``) as 400. Errors are classified by +wording: "not attached" and "not found" mean missing, "already" / "in use" / +"attached" / "mapped" mean conflict. Objects are addressed by id (UUID) for volumes and hosts. Volume identity: ``wwn`` is returned as ``naa.``, ``nguid`` as ``nguid.<32 hex>``; this adapter reports them as ``naa.`` / ``eui.`` (lowercase). +Mappings: ``host_volume_mapping.logical_unit_number`` is the SCSI LUN. For +NVMe hosts the namespace id is the volume's own ``nsid`` field ("NVMe +Namespace unique identifier in the NVME subsystem"), so a mapping to a host +whose initiators are all ``NVMe`` reports that ``nsid`` (falling back to +``logical_unit_number`` when the array returns none). + Sizes are bytes; PowerStore requires a multiple of 8 KiB (8192) and at least 1 MiB. +Checked against vendor source +----------------------------- +(PyPowerStore = github.com/dell/python-powerstore, gopowerstore = +github.com/dell/gopowerstore, csi-powerstore = github.com/dell/csi-powerstore) + +* Session: ``auth_cookie`` cookie and ``DELL-EMC-TOKEN`` header from + ``GET login_session``, both sent on later requests, logout is + ``POST /logout`` (PyPowerStore ``client.py`` ``AuthenticationManager``); + gopowerstore re-logs in on 403 (``api/api.go`` ``Query``). +* Create answers ``{"id"}`` (gopowerstore ``CreateResponse``), modify / + delete / attach / detach an empty body (``EmptyResponse``); attach and + detach are ``POST volume/{id}/attach|detach {"host_id"}`` (PyPowerStore + ``constants.py`` ``MAP_VOLUME_TO_HOST_URL``). +* Error body ``messages[].message_l10n`` (gopowerstore ``api/api.go`` + ``ErrorMsg``); status codes per gopowerstore ``gopowerstore_types.go``: + name in use 422 or 500, delete of an attached volume 422, detach of a + non-attached volume / missing host 400 or 404; wording "Host is not + attached to volume" and "already registered with another host" + (csi-powerstore ``controller/base.go``, ``node/node.go``). +* Host fields ``host_initiators`` (GET), ``initiators`` (create), + ``add_initiators`` (modify), each ``{"port_name", "port_type"}`` with + ``port_type`` ``iSCSI``/``FC``/``NVMe`` (gopowerstore ``host_types.go``); + NVMe/TCP and NVMe/FC hosts both register their NQN as ``NVMe``, FC WWPNs + colon-separated lowercase (csi-powerstore ``buildInitiatorsArray``, + ``formatWWPN``). +* Volume ``nsid`` field (gopowerstore ``volume_types.go``) and embedded + selects on ``host_volume_mapping`` (``HostVolumeMapping.Fields``). +* ``name=eq.`` unquoted (gopowerstore ``GetHostByName``, + ``GetVolumeByName``). +* Size: multiple of 8192, minimum 1 MB (gopowerstore ``VolumeCreate``, + csi-powerstore ``VolumeSizeMultiple`` / ``MinVolumeSizeBytes``). + Verify against a real system ---------------------------- -* The session cookie name (``auth_cookie``) and that the CSRF header is - exactly ``DELL-EMC-TOKEN`` on both the login response and requests. -* That ``POST /volume`` and ``POST /host`` answer ``201 {"id": ...}`` and that - PATCH/DELETE/attach/detach answer ``204`` with an empty body. -* The status code (422 assumed) and ``message_l10n`` wording for: duplicate - name, initiator already registered to another host, delete of an attached - volume/host, shrink, detach of a non-attached volume. Classification below - is by wording ("already", "in use", "attached", "mapped", "not found"). -* The host field holding initiators on GET is ``host_initiators`` - (create uses ``initiators``, modify ``add_initiators``), each - ``{"port_name", "port_type"}``; ``port_type`` values ``iSCSI``/``FC``/ - ``NVMe`` (NVMe/TCP and NVMe/FC both ``NVMe``), and that FC ``port_name`` is - accepted/returned colon-separated lowercase (``21:00:...``). -* That ``logical_unit_number`` in ``host_volume_mapping`` is the NVMe NSID for - NVMe hosts, and which LUN the array picks first (fake: 1). -* That a ``name=eq.`` filter needs no quoting for host names containing - dots or commas. -* That the size multiple is 8192 bytes on 4.x as on 3.x. +* Status and wording for a shrink, deleting a host with mapped volumes, and + re-adding an initiator already on the same host. +* Whether ``logical_unit_number`` on an NVMe host's mapping equals the + volume's ``nsid`` (the adapter prefers ``nsid``), and which LUN the array + picks first for SCSI (fake: 1). +* Exact behaviour of ``name=eq.`` for names containing commas or parentheses + (simplyblock names are ``sb-``; host names are caller-chosen). """ from __future__ import annotations @@ -73,10 +105,12 @@ TOKEN_HEADER = "DELL-EMC-TOKEN" VOLUME_FIELDS = "id,name,size,wwn,nguid" HOST_FIELDS = "id,name,host_initiators" -MAPPING_FIELDS = "id,host_id,host_group_id,volume_id,logical_unit_number" +MAPPING_FIELDS = ("id,host_id,host_group_id,volume_id,logical_unit_number," + "host(host_initiators),volume(nsid)") _PAGE = 1000 -_NOT_FOUND = ("not found", "does not exist", "not exist", "no such") +# "not attached": detach of a volume that is not (or no longer) attached. +_NOT_FOUND = ("not found", "does not exist", "not exist", "no such", "not attached") _CONFLICT = ("already", "in use", "exists", "attached", "mapped", "is used", "duplicate") @@ -132,6 +166,14 @@ def _port_key(port_type: str, port_name: str) -> str: return port_name +def _is_nvme_host(host: Any) -> bool: + """True for an embedded ``host`` whose initiators are all NVMe.""" + ports = host.get("host_initiators") if isinstance(host, dict) else None + if not ports: + return False + return all(p.get("port_type") == "NVMe" for p in ports) + + class DellPowerStoreAdapter(ArrayAdapter): vendor = "dell-powerstore" display_name = "Dell PowerStore" @@ -189,9 +231,10 @@ def close(self) -> None: def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, json: Any = None, missing_ok: bool = False) -> requests.Response | None: + # 500 included: a duplicate volume name may come back as 500, not 422. resp = self.session.request(method, f"{API}{path}", params=params, json=json, - ok=(400, 404, 422)) - if resp.status_code in (400, 404, 422): + ok=(400, 404, 422, 500)) + if resp.status_code in (400, 404, 422, 500): err = _classify(method, path, resp) if missing_ok and isinstance(err, ArrayNotFoundError): return None @@ -307,6 +350,8 @@ def _mappings(self, volume_id: str) -> list[Mapping]: host = i.get("host_id") or i.get("host_group_id") if host: lun = i.get("logical_unit_number") + if _is_nvme_host(i.get("host")): + lun = (i.get("volume") or {}).get("nsid") or lun out.append(Mapping(volume_id=i["volume_id"], host_id=host, lun=int(lun) if lun is not None else None)) return out diff --git a/simplyblock_core/storage_arrays/adapters/hitachi_vsp.py b/simplyblock_core/storage_arrays/adapters/hitachi_vsp.py index 29cead34cd..8d3c14c9eb 100644 --- a/simplyblock_core/storage_arrays/adapters/hitachi_vsp.py +++ b/simplyblock_core/storage_arrays/adapters/hitachi_vsp.py @@ -35,38 +35,61 @@ Volume identity: SCSI ``naa.`` (lowercase). +Checked against vendor source (Hitachi's OpenStack Cinder driver +``cinder/volume/drivers/hitachi/hbsd_rest.py`` / ``hbsd_rest_api.py`` and the +Ansible collection ``hitachivantara.vspone_block``, ``plugins/module_utils/ +gateway/gateway_manager.py``, ``vsp_volume.py``, ``vsp_host_group_gateway.py``, +``vsp_rg_lock_gateway.py``, ``common/vsp_constants.py``, ``vsp_utils.py``): + +* Sessions: ``POST .../sessions`` with basic auth -> ``token``/``sessionId``, + ``Authorization: Session ``, ``DELETE .../sessions/{id}``; 401 means + re-login. Jobs: 202 + ``jobId``, done at ``status == "Completed"``, + ``state == "Succeeded"``, new object = last segment of + ``affectedResources[0]``; a failed job's ``error`` is the usual body. +* ``POST /ldevs {poolId, blockCapacity}`` without ``ldevId`` auto-assigns + the number (``startLdevId``/``endLdevId`` restrict it); the label is set + afterwards with ``PUT /ldevs/{id} {"label"}``, exactly as Cinder does. + Expand: ``POST .../actions/expand/invoke {"parameters": + {"additionalBlockCapacity"}}``. A deleted LDEV reads ``emulationType == + "NOT DEFINED"``. +* Lookup by label: there is no label filter; Cinder scans ``GET /ldevs? + ldevOption=dpVolume&headLdevId=&count=16384`` stepping the head by + ``count`` up to LDEV 65535 and ignoring KART40044-E (window past the + end). This adapter does the same (it used to stop after the first short + page, which missed LDEVs above the first 16k numbers). +* NAA: derived from the storage serial and the LDEV number (Ansible + ``NAIDCalculator``), so a reused LDEV number has the same WWN. +* ``GET /ldevs/{id}`` lists LU paths under ``ports`` (``portId``, + ``hostGroupNumber``, ``hostGroupName``, ``lun``); ``GET /host-wwns`` / + ``/host-iscsis`` take ``portId`` + ``hostGroupNumber``; ``POST + /host-groups {portId, hostGroupName, hostMode[, hostModeOptions]}``; + ``POST /luns {portId, hostGroupNumber, ldevId, lun}``; ``DELETE + /luns/{port},{gid},{lun}``. +* Resource-group lock: ``.../resource-group-service/actions/lock/invoke`` + with ``{"parameters": {"waitTime"}}`` lives under + ``/ConfigurationManager/v1/services`` (embedded) or + ``/ConfigurationManager/v1/{storageDeviceId}/services`` (REST server), + not under ``/objects`` (fixed here). Ansible POSTs it, Cinder PUTs it. +* Error codes (classified before the text fallback): KART30013-E and + ``EX_ENOOBJ`` = not found; KART40050-E..52-E, SSB 2E11/2205, 2E11/2207, + ``EX_EACCES`` = resource group locked; SSB 2E22/0001 (LDEV already + defined), B958/015A (LU path already defined), B958/0947 (another LDEV on + that LUN) = conflict. Storage failures carry SSB1/SSB2 in ``errorCode``. + Verify against a real system: -* ``POST /ldevs`` without ``ldevId`` auto-assigns a free LDEV number (newer - API versions); older microcode may require ``ldevId`` or a - ``startLdevId``/``endLdevId`` range -- set ``ldev_range`` there. -* The label is set in a second request (``PUT /ldevs/{id}``); a crash between - the two leaves an unlabeled LDEV the client cannot find (a label failure - is cleaned up here, a crash is not). -* ``find_volume`` lists DP-VOLs of the pool (``ldevOption=dpVolume&poolId=``, - paged by ``headLdevId``/``count``) and matches labels client-side: one or a - few requests per 16k LDEVs. The API has no label filter that we know of. -* ``naaId`` is derived from the serial and LDEV number, so a deleted LDEV's - number reused by a later volume has the same WWN; ownership then rests on - the ``sb-`` label. -* DP-VOL capacity: the adapter rounds to 1 MiB (safe for data-reduction - volumes, which need MiB alignment) and creates at least 64 MiB (DP-VOLs - have a ~47 MB floor on some models). -* ``GET /ldevs/{id}`` lists its LU paths under ``ports`` (``portId``, - ``hostGroupNumber``, ``hostGroupName``, ``lun``); mappings are read from - there. -* ``GET /host-wwns`` and ``/host-iscsis`` need ``portId`` and - ``hostGroupNumber``, so detecting an initiator owned by another host lists - every host group on each configured port (one request per group). - FC rejects a WWN registered to two host groups of one port; iSCSI allows - the same initiator in several targets, so the scan is what enforces - ownership there. -* Some models / configurations require the resource-group lock - (``services/resource-group-service/actions/lock/invoke``) around changes; - enable with ``use_lock``. -* Error ``messageId``s: storage-side failures mostly arrive as - ``KART30000-E`` with SSB codes in ``errorCode``; errors are classified by - HTTP status first, then by message text (see ``_classify``). +* Older microcode that may require ``ldevId`` on ``POST /ldevs`` (set + ``ldev_range`` there). +* A crash between LDEV create and label leaves an unlabeled LDEV the client + cannot find (a label failure is cleaned up here, a crash is not). +* DP-VOL capacity: the adapter rounds to 1 MiB and creates at least 64 MiB + (Cinder only creates whole GiB); the real floor per model. +* Whether every model returns ``naaId`` (Ansible computes it when absent; + this adapter then reports no WWN). +* FC refusing a WWN in two host groups of one port and iSCSI allowing an + initiator in several targets (the ownership scan does not rely on either). +* Which models need the resource-group lock (``use_lock``); Cinder retries + lock errors for up to two hours, this adapter reports a conflict. * NVMe-oF (``/nvm-subsystems``) is not implemented: only FC and iSCSI. """ from __future__ import annotations @@ -93,12 +116,17 @@ ) from ..http import json_of +API = "/ConfigurationManager/" ROOT = "/ConfigurationManager/v1/objects" BLOCK = 512 NOT_DEFINED = "NOT DEFINED" MAX_LUN = 2047 -#: ``count`` per ``GET /ldevs`` page (the API maximum). +#: ``count`` per ``GET /ldevs`` page (the API maximum). ``headLdevId`` steps +#: by this much, as in Hitachi's Cinder driver (``hbsd_rest.py``). PAGE_SIZE = 16384 +#: Highest LDEV number scanned when no ``ldev_range`` is set (Cinder's +#: ``_MAX_LDEV_ID``). +MAX_LDEV_ID = 65535 _PORT_RE = re.compile(r"^CL[0-9A-Z]+-[A-Z0-9]+$") _RANGE_RE = re.compile(r"^\s*(\d+)\s*-\s*(\d+)\s*$") @@ -107,6 +135,22 @@ #: (401/403 stay with RestSession -> ArrayAuthError). _OWN_STATUSES = (400, 404, 405, 409, 412, 500, 502, 503, 504) +# Structured codes, from Hitachi's Cinder driver (hbsd_rest_api.py, hbsd_rest.py). +#: messageId of "the specified object does not exist". +_MSGID_NOT_FOUND = frozenset({"KART30013-E"}) +#: messageIds of a resource-group lock held by someone else. +_MSGID_LOCKED = frozenset({"KART40050-E", "KART40051-E", "KART40052-E"}) +#: (SSB1, SSB2) pairs: LDEV already defined, LU path already defined, another +#: LDEV already mapped to that LUN, resource group locked. +_SSB_CONFLICT = frozenset({("2E22", "0001"), ("B958", "015A"), ("B958", "0947"), + ("2E11", "2205"), ("2E11", "2207")}) +#: errorCode.errorCode values. +_RC_NOT_FOUND = frozenset({"EX_ENOOBJ"}) +_RC_CONFLICT = frozenset({"EX_EACCES"}) +#: A ``GET /ldevs`` window beyond the array's last LDEV number. +_MSGID_BAD_RANGE = "KART40044-E" +_RC_BAD_RANGE = ("EX_ENLDEV", "EX_INVARG") + _CONFLICT_TEXT = ("already", "in use", "is used", "registered", "duplicat", "lu path is defined", "lu paths are defined", "locked") _MISSING_TEXT = ("does not exist", "not found", "not defined", "not exist") @@ -133,6 +177,16 @@ def _classify(where: str, status: int | None, body: Any) -> ArrayError: if status: text += f" [HTTP {status}]" lowered = message.lower() + if isinstance(body, dict): + raw = body.get("errorCode") + err: dict[str, Any] = raw if isinstance(raw, dict) else {} + ssb_pair = (str(err.get("SSB1", "")).upper(), str(err.get("SSB2", "")).upper()) + rc = str(err.get("errorCode", "")) + mid = str(body.get("messageId") or "") + if mid in _MSGID_NOT_FOUND or rc in _RC_NOT_FOUND: + return ArrayNotFoundError(text) + if mid in _MSGID_LOCKED or ssb_pair in _SSB_CONFLICT or rc in _RC_CONFLICT: + return ArrayConflictError(text) if status == 404: return ArrayNotFoundError(text) if status in (409, 412): @@ -205,6 +259,12 @@ def __init__(self, ctx): self._range = (int(m.group(1)), int(m.group(2))) sdid = self.option("storage_device_id") self._base = f"{ROOT}/storages/{sdid}" if sdid else ROOT + # Services are not under /objects: /ConfigurationManager/v1/services + # (embedded, Hitachi Ansible vsp_rg_lock_gateway.py) or + # /ConfigurationManager/v1/{storageDeviceId}/services (REST API + # server, Hitachi Cinder hbsd_rest_api.py ``service_url``). + self._services = (f"{API}v1/{sdid}/services" if sdid + else f"{API}v1/services") self._session_id: Any = None self._port_types: dict[str, Protocol | None] = {} @@ -235,7 +295,7 @@ def close(self) -> None: def _req(self, method: str, path: str, *, params: Any = None, json: Any = None) -> tuple[int, Any]: - full = path if path.startswith(ROOT) else f"{self._base}{path}" + full = path if path.startswith(API) else f"{self._base}{path}" resp = self.session.request(method, full, params=params, json=json, ok=_OWN_STATUSES) try: @@ -288,7 +348,7 @@ def _change(self) -> Iterator[None]: if not self.option("use_lock"): yield return - svc = "/services/resource-group-service/actions" + svc = f"{self._services}/resource-group-service/actions" self._run("POST", f"{svc}/lock/invoke", {"parameters": {"waitTime": int(self.option("lock_wait_s"))}}) try: @@ -361,29 +421,40 @@ def get_volume(self, volume_id: str) -> Volume | None: return self._volume(ldev) if ldev else None def find_volume(self, name: str) -> Volume | None: - head, end = self._range if self._range else (0, None) - found: list[dict[str, Any]] = [] - while True: - page = self._data("/ldevs", ldevOption="dpVolume", poolId=self._pool, - headLdevId=head, count=PAGE_SIZE) - for ldev in page: - if end is not None and int(ldev["ldevId"]) > end: + # ``count`` is a window of LDEV numbers starting at ``headLdevId``, so + # step the head by ``count`` (not by the last id returned) as Hitachi's + # Cinder driver does; a window past the array's last LDEV number + # answers KART40044-E / EX_ENLDEV and ends the scan. ``poolId`` is not + # combined with ``headLdevId`` (no vendor client does); the pool is + # checked on each entry instead. + start, end = self._range if self._range else (0, MAX_LDEV_ID) + found: dict[int, dict[str, Any]] = {} + head = start + while head <= end: + count = min(PAGE_SIZE, end - head + 1) + try: + page = self._data("/ldevs", ldevOption="dpVolume", headLdevId=head, + count=count) + except ArrayError as e: + past_end = _MSGID_BAD_RANGE in str(e) or any( + rc in str(e) for rc in _RC_BAD_RANGE) + if past_end and head > start: # never on the first window break - if ldev.get("label") == name and self._defined(ldev): - found.append(ldev) - if len(page) < PAGE_SIZE: - break - head = int(page[-1]["ldevId"]) + 1 - if end is not None and head > end: - break + raise + for ldev in page: + lid = int(ldev["ldevId"]) + if (start <= lid <= end and ldev.get("label") == name + and self._defined(ldev) + and int(ldev.get("poolId", self._pool)) == self._pool): + found[lid] = ldev + head += count if len(found) > 1: raise ArrayConflictError( - f"Several LDEVs labeled {name!r}: " - + ", ".join(str(f["ldevId"]) for f in found)) + f"Several LDEVs labeled {name!r}: " + ", ".join(str(i) for i in sorted(found))) if not found: return None # List entries may omit fields; read the LDEV itself. - return self.get_volume(str(found[0]["ldevId"])) + return self.get_volume(str(next(iter(found)))) def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: ldev = self._require(volume_id) diff --git a/simplyblock_core/storage_arrays/adapters/hpe_nimble.py b/simplyblock_core/storage_arrays/adapters/hpe_nimble.py index 35cd1583ea..e2036c5c98 100644 --- a/simplyblock_core/storage_arrays/adapters/hpe_nimble.py +++ b/simplyblock_core/storage_arrays/adapters/hpe_nimble.py @@ -26,32 +26,63 @@ reports it as ``wwn="eui."`` (``Volume.wwn`` holds the SCSI designator, NAA or EUI-64), so the identifier equals the host's sysfs wwid. +Asynchronous jobs: a mutating request may answer **202** with +``{"messages": [{"code": "SM_async_job_id", "arguments": {"job_id": ...}}]}`` +instead of the object; the adapter then polls ``GET /v1/jobs/`` until +``state`` is ``done`` (as the NimbleOS SDK does) and re-reads what it needs. + +Checked against vendor source +----------------------------- +(HPE ``nimble-python-sdk``: ``nimbleclient/v1/restclient.py``, the +``nimbleclient/v1/api/*.py`` field docs and ``tests/``; Cinder +``cinder/volume/drivers/hpe/nimble.py``; HPE ``common-host-libs`` +``linux/device.go``.) + +* Tokens: ``POST /v1/tokens {"data": {username, password}}`` -> + ``data.session_token`` / ``data.id``; ``X-Auth-Token``; release with + ``DELETE /v1/tokens/``; bad logins and expired tokens answer + ``SM_http_unauthorized``. +* ``{"data": ...}`` envelopes; errors ``{"messages": [...]}`` where a generic + ``SM_http_`` entry precedes the specific one (Cinder reads + ``messages[1]``). Specific codes seen in vendor code: ``SM_eexist`` (with + ``SM_http_conflict``) for duplicates, ``SM_enoent`` / ``SM_http_not_found`` + for unknown objects, ``SM_vol_size_decreased`` (shrink without ``force``; + we never send ``force``), ``SM_vol_has_clone``. Older NimbleOS ("fiji and + below") answers ``SM_http_bad_request`` (400) for an unknown id, hence + ``get_volume`` treats 400 as missing. +* Sizes in MiB; ``serial_number`` is the volume's SCSI identifier; arrays + report ``model``/``serial``/``version``. +* Offline before delete (``PUT {"online": false}``). Offlining a volume that + is already offline fails with "Object is already in requested state"; + ``delete_volume`` tolerates that, as Cinder does, so a retried delete works. +* Query filters as plain parameters (``?vol_id=...``, + ``?initiator_group_id=...``, ``?name=...``) returning an empty ``data`` + list when nothing matches (Cinder ``get_query`` callers). +* ``label`` is required for iSCSI initiators; FC WWPNs in colon form (Cinder + ``_format_to_wwpn``); ACRs with ``apply_to: "both"``. +* ACR ``lun``: 0 for iSCSI, 0-2047 for FC (SDK field doc); Cinder uses LUN 0 + unless group-scoped targets are enabled. +* Multipath shows an EUI-64 device as WWID ``2``: common-host-libs + strips the leading type digit (2 EUI, 3 NAA) to compare with the serial + the CSP returns. + Verify against a real system ---------------------------- -* The EUI-64 derivation above (``eui.`` + ``serial_number`` lowercase), both - for iSCSI and FC, on NimbleOS 5.x/6.x. -* Error codes: ``SM_http_unauthorized`` (401) for bad login and expired - tokens; ``SM_eexist`` (409) for duplicate names and ACRs; - ``SM_http_not_found``/``SM_enoent`` (404) for unknown ids; ``SM_vol_online`` - (409, deleting an online volume); ``SM_vol_has_acl`` / ACR-related refusal - when offlining a connected volume; ``SM_invalid_arg`` (400). The adapter - keys on HTTP status, and treats 400 + ``SM_enoent`` / "not found" as - not-found. -* Whether ``GET /v1/volumes/`` for a malformed id answers 400 or 404 - (both are treated as "missing" by ``get_volume``). -* Query filtering on any attribute (``/v1/initiators/detail?iqn=...``, - ``/v1/access_control_records/detail?vol_id=...``) and that a filtered list - with no hits returns ``data: []`` rather than 404. -* LUN assignment: with volume-scoped iSCSI targets (the default on recent - NimbleOS) every iSCSI ACR shows ``lun`` 0; group-scoped targets and FC get - distinct LUNs per initiator group. -* That ``label`` is required for iSCSI initiators (max 64 chars) and is - unique only within its group. -* Shrinking (``PUT size`` below current) is refused; we never request it. +* That this CSP serial is the volume's ``serial_number`` on NimbleOS 5.x/6.x + for iSCSI and FC, i.e. ``eui.`` + lowercase ``serial_number`` equals the + sysfs ``wwid`` (the nimble-csp source is not published). +* Status and code of the refusals the fake invents: deleting an online + volume (``SM_vol_online``?), offlining or deleting a volume that has ACRs, + deleting an initiator group with ACRs, offlining an offline volume. +* Which operations answer 202 on a real array, and the ``result`` of a + failed job (the SDK only waits for ``state == "done"``). +* ``label`` length limit (64?) and uniqueness scope; whether an initiator may + sit in two groups (not relied on: the adapter checks ownership first). """ from __future__ import annotations import hashlib +import math from typing import Any from urllib.parse import quote @@ -77,6 +108,9 @@ from ..http import body_text, json_of MIB = 1 << 20 +_JOB_POLL_S = 2.0 +#: markers of "Object is already in requested state" (offlining an offline volume) +_ALREADY = ("SM_ealready", "already in requested state") _NOT_FOUND_CODES = frozenset({"SM_enoent", "SM_http_not_found", "SM_no_such_object"}) @@ -99,6 +133,8 @@ class HpeNimbleAdapter(ArrayAdapter): "array's default pool)."), OptionDoc("performance_policy", "Performance policy name for new volumes " "(default: the array default)."), + OptionDoc("job_timeout_s", "How long to wait for an asynchronous (HTTP 202) job.", + default=300), ) def __init__(self, ctx): @@ -158,10 +194,33 @@ def _request(self, method: str, path: str, *, params: Any = None, json: Any = No raise ArrayRequestError(msg, status=status) def _data(self, method: str, path: str, **kw: Any) -> Any: + """The response's ``data``; ``None`` for a missing object (with + ``missing_ok``) or after waiting for an asynchronous job, in which + case the caller re-reads the object.""" resp = self._request(method, path, **kw) if resp is None: return None - return (json_of(resp) or {}).get("data") + body = json_of(resp) or {} + if resp.status_code == 202: + self._wait_job(body, f"{method} {path}") + return None + return body.get("data") + + def _wait_job(self, body: dict[str, Any], what: str) -> None: + job_id = None + for m in body.get("messages") or []: + if m.get("code") == "SM_async_job_id": + job_id = (m.get("arguments") or {}).get("job_id") + if not job_id: + return # 202 without a job: nothing to wait for + polls = max(1, math.ceil(float(self.option("job_timeout_s")) / _JOB_POLL_S)) + for _ in range(polls): + job = self._data("GET", f"/v1/jobs/{quote(str(job_id), safe='')}") or {} + if job.get("state") == "done": + return + self.session.sleep(_JOB_POLL_S) + raise ArrayRequestError(f"{what}: job {job_id} did not finish within " + f"{self.option('job_timeout_s')} s") def _list(self, path: str, **filters: str) -> list[dict[str, Any]]: return list(self._data("GET", path, params=filters) or []) @@ -203,7 +262,11 @@ def get_info(self) -> ArrayInfo: def create_volume(self, name: str, size_bytes: int) -> Volume: body = {"name": name, "size": -(-size_bytes // MIB), **self._volume_extra()} - return self._volume(self._data("POST", "/v1/volumes", json={"data": body})) + data = self._data("POST", "/v1/volumes", json={"data": body}) + vol = self._volume(data) if data else self.find_volume(name) + if vol is None: + raise ArrayRequestError(f"Volume {name} not found right after creation") + return vol def get_volume(self, volume_id: str) -> Volume | None: try: @@ -222,12 +285,22 @@ def find_volume(self, name: str) -> Volume | None: def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: data = self._data("PUT", f"/v1/volumes/{quote(volume_id, safe='')}", json={"data": {"size": -(-size_bytes // MIB)}}) - return self._volume(data) + if data: + return self._volume(data) + vol = self.get_volume(volume_id) + if vol is None: + raise ArrayNotFoundError(f"Volume {volume_id} vanished during resize") + return vol def delete_volume(self, volume_id: str) -> None: path = f"/v1/volumes/{quote(volume_id, safe='')}" - self._data("PUT", path, json={"data": {"online": False}}) - self._request("DELETE", path) + try: + self._data("PUT", path, json={"data": {"online": False}}) + except (ArrayRequestError, ArrayConflictError) as e: + # A retried delete finds the volume already offline. + if not any(marker in str(e) for marker in _ALREADY): + raise + self._data("DELETE", path) # -- hosts (initiator groups) ------------------------------------------- @@ -275,7 +348,11 @@ def ensure_host(self, spec: HostSpec) -> str: if group is None: group = self._data("POST", "/v1/initiator_groups", json={"data": {"name": spec.name, "access_protocol": protocol, - "description": "simplyblock"}}) + "description": "simplyblock"}}) \ + or self._group(spec.name) + if group is None: + raise ArrayRequestError(f"Initiator group {spec.name} not found right " + "after creation") group_id = str(group["id"]) for value in wanted: body: dict[str, Any] = {"initiator_group_id": group_id, @@ -286,8 +363,8 @@ def ensure_host(self, spec: HostSpec) -> str: return group_id def delete_host(self, host_id: str) -> None: - self._request("DELETE", f"/v1/initiator_groups/{quote(host_id, safe='')}", - missing_ok=True) + self._data("DELETE", f"/v1/initiator_groups/{quote(host_id, safe='')}", + missing_ok=True) # -- mappings (access control records) ----------------------------------- @@ -309,13 +386,18 @@ def map_volume(self, volume_id: str, host_id: str) -> Mapping: acr = self._data("POST", "/v1/access_control_records", json={"data": {"vol_id": volume_id, "initiator_group_id": host_id, "apply_to": "both"}}) - return self._mapping(acr) + if acr: + return self._mapping(acr) + existing = self._acrs(volume_id, host_id) + if not existing: + raise ArrayRequestError(f"ACR {volume_id} -> {host_id} not found after creation") + return self._mapping(existing[0]) def unmap_volume(self, volume_id: str, host_id: str) -> None: for acr in self._acrs(volume_id, host_id): - self._request("DELETE", - f"/v1/access_control_records/{quote(str(acr['id']), safe='')}", - missing_ok=True) + self._data("DELETE", + f"/v1/access_control_records/{quote(str(acr['id']), safe='')}", + missing_ok=True) def list_mappings(self, volume_id: str) -> list[Mapping]: return [self._mapping(a) for a in self._acrs(volume_id)] diff --git a/simplyblock_core/storage_arrays/adapters/hpe_wsapi.py b/simplyblock_core/storage_arrays/adapters/hpe_wsapi.py index 9642886ec8..5cff6bf3d6 100644 --- a/simplyblock_core/storage_arrays/adapters/hpe_wsapi.py +++ b/simplyblock_core/storage_arrays/adapters/hpe_wsapi.py @@ -26,33 +26,61 @@ so entries are de-duplicated by (volume, host). Only iSCSI and FC hosts are supported: NVMe/TCP and NVMe/FC hosts exist on -Alletra MP and Primera but their WSAPI host fields (``NVMETCPPaths``/NQN -attributes) are not verified, so NVMe is refused up front. +Alletra MP and Primera (python-3parclient's ``createHost`` sends an ``NQN`` +field) but NVMe host paths and exports are not implemented here, so NVMe is +refused up front. + +Checked against vendor source +----------------------------- +(python-3parclient ``hpe3parclient/client.py``/``http.py`` and its WSAPI mock +``test/HPE3ParMockServer_flask.py``; HPE ``hpe_storage_flowkit_py`` +``v1/core/rest_client.py``/``session.py``/``utils/volume_utils.py``, the +library behind Cinder's Alletra MP driver; Cinder ``hpe_3par_common.py``, +``hpe_3par_iscsi.py``; HPE ``common-host-libs`` ``linux/device.go``.) + +* Login/logout, the session header, ``{"code", "desc"}`` errors. +* Error codes: 5 INV_USER_PASS (403 on login), 16 EXISTENT_HOST (409), 17 + NON_EXISTENT_HOST (404), 19 NON_EXISTENT_VLUN (404), 22 EXISTENT_VOL + (409), 23 NON_EXISTENT_VOL (404), 26 EXPORTED_VLUN (409, host still + exported), 34 "resource in use" (409, e.g. volume in a VV set or + exported), 73 EXISTENT_PATH (409). Code 29 is *not* a VLUN conflict: + Cinder sees it as HTTP 400 on a volume delete, so it is no longer treated + as a conflict. +* 403 is also used for non-auth refusals (PERM_DENIED, 150 cannot grow this + volume type, 153 new size exceeds the CPG limit, 120 volume in remote copy, + VV_SIZE_CANNOT_REDUCE, ...). python-3parclient still re-authenticates once + on any 401/403, so a 403 stays an auth error except for the listed + non-auth codes (``_FORBIDDEN_NOT_AUTH``), which fail directly. +* GROW (``action`` 3) takes the increment in MiB; minimum create size is + 256 MiB, name limit 31 characters (vendor mock). +* ``reduce: true`` is the dedup+compression create key on Primera *and* + Alletra MP (python-3parclient maps ``tdvv``+``compression`` to it when the + WSAPI is Primera-class; flowkit sends ``tpvv``/``reduce``); ``tpvv: true`` + is thin. 3PAR (pre-Primera) dedup/compression is not offered. +* ``POST /hosts`` and ``PUT /hosts`` refuse ``iSCSINames`` together with + ``FCWWNs`` (INV_INPUT_PARAM_CONFLICT), hence one path type per request; a + host may carry both types (Cinder's iSCSI driver adds iSCSI names to an + existing host that has none). +* VLUN query ``"volumeName EQ "`` (quotes included), ``members`` list, + template VLUNs ``active: false`` plus active per-path VLUNs; the + ``Location`` header is ``/api/v1/vluns/,,``. +* Personas: 1 Generic, 2 Generic-ALUA, 8 VMware, 11 WindowsServer. +* Volume ``wwn`` field; HPE host tooling shows 3PAR-family WWIDs as + ``naa.60002ac...``. Verify against a real system ---------------------------- -* Error codes (taken from python-3parclient / Cinder usage, not from a - spec): 23 NON_EXISTENT_VOL (404), 22 EXISTENT_VOL (409), 17 - NON_EXISTENT_HOST (404), 16 EXISTENT_HOST (409), 73 EXISTENT_PATH -- WWN/IQN - already owned by another host (409), 19 NON_EXISTENT_VLUN (404), 29 - EXISTENT_VLUN (409), 34 VOLUME_HAS_VLUNS / 26 HOST_IN_USE (409), 5 invalid - login (403), 6 invalid/expired session key (401). The adapter keys its - behaviour on HTTP status first and on codes only where listed in - ``_NOT_FOUND_CODES`` / ``_CONFLICT_CODES``. -* Whether a 403 is ever returned for a non-auth reason (it would trigger a - pointless re-login before failing). -* Size granularity: 3PAR/Primera allocate in 256 MiB units and report the - rounded ``sizeMiB``; we therefore round to 256 MiB (min 256 MiB) so the - requested and reported sizes agree. Alletra MP may accept 1 MiB units. -* ``reduce: true`` (dedup + compression) as the Alletra MP create flag, versus - ``tdvv``/``compression`` on Primera; ``tpvv: true`` for thin on 3PAR/Primera. -* Whether one host may carry both iSCSI and FC paths (passed through as given). -* The exact ``query`` syntax (``"volumeName EQ x"``, double quotes included) - and that it returns ``{"total", "members"}`` with an empty list (not 404) - when nothing matches; whether active VLUN entries carry ``active: true``. -* Persona 2 = "Generic-ALUA" (persona 1 = Generic, 11 = VMware). +* The status/code for an expired or unknown session key (the fake uses 401 + code 6); python-3parclient re-authenticates on both 401 and 403. +* Size granularity on Alletra MP (the 256 MiB step is from the vendor mock + and growVolume's "rounded up to the next chunklet size (e.g. 256 or 1000 + MiB)"); we round to 256 MiB so requested and reported sizes agree. +* A filtered VLUN query with no hits answering 200 with an empty + ``members`` list (python-3parclient implies it but cannot tell it apart + from a 404) and the status/code for a duplicate VLUN. * Max session count per user (stale keys from crashed processes may exhaust - it); we always ``DELETE`` the key on close. + it; flowkit reuses one session per user and drops it after 14 minutes); + we always ``DELETE`` the key on close. """ from __future__ import annotations @@ -89,7 +117,11 @@ PATH_ADD = 1 _NOT_FOUND_CODES = frozenset({17, 19, 23}) -_CONFLICT_CODES = frozenset({16, 22, 26, 29, 34, 73}) +_CONFLICT_CODES = frozenset({16, 22, 26, 34, 73}) +#: 403 codes that are refusals, not session/credential problems: 120 volume +#: in remote copy, 150 cannot grow this volume type, 153 new size exceeds the +#: CPG limit, 215 remote copy group already started. +_FORBIDDEN_NOT_AUTH = frozenset({120, 150, 153, 215}) _LOCATION_RE = re.compile(r"/api/v1/vluns/(?P[^,]+),(?P\d+),(?P[^,/]*)") @@ -111,8 +143,8 @@ class HpeWsapiAdapter(ArrayAdapter): option_docs = ( OptionDoc("cpg", "Common Provisioning Group new volumes are created in.", required=True), - OptionDoc("provisioning", "'tpvv' (thin, 3PAR/Primera) or 'reduce' " - "(thin + dedup + compression, Alletra MP).", default="tpvv"), + OptionDoc("provisioning", "'tpvv' (thin) or 'reduce' (thin + dedup + " + "compression, Primera / Alletra MP).", default="tpvv"), OptionDoc("persona", "Host persona id for new hosts (2 = Generic-ALUA).", default=2), ) @@ -171,6 +203,8 @@ def _call(self, method: str, path: str, *, params: Any = None, json: Any = None, pass msg = f"{method} {path} -> HTTP {status}" + (f" code {code}" if code is not None else "") \ + f": {desc}" + if status == 403 and code in _FORBIDDEN_NOT_AUTH: + raise ArrayRequestError(msg, status=status) if status in (401, 403): raise ArrayAuthError(msg) if status == 404 or code in _NOT_FOUND_CODES: @@ -248,8 +282,8 @@ def ensure_host(self, spec: HostSpec) -> str: wwns = [w.upper() for w in spec.addresses(Protocol.FC)] host = self._host(spec.name) if host is None: - # Create with one path type; any second type is added below - # (whether WSAPI accepts both in one request is unverified). + # Create with one path type (WSAPI refuses iSCSINames together + # with FCWWNs: INV_INPUT_PARAM_CONFLICT); a second type is added below. body: dict[str, Any] = {"name": spec.name, "persona": int(self.option("persona"))} if iqns: body["iSCSINames"] = iqns @@ -263,8 +297,7 @@ def ensure_host(self, spec: HostSpec) -> str: have_wwns = {str(p.get("wwn", "")).upper() for p in host.get("FCPaths") or []} add_iqns = [i for i in iqns if i.lower() not in have_iqns] add_wwns = [w for w in wwns if w not in have_wwns] - # One path type per request (whether WSAPI accepts both in one - # modify is unverified; separate requests are always valid). + # One path type per request: a modify with both is refused too. if add_iqns: self._call("PUT", f"/hosts/{quote(spec.name, safe='')}", json={"pathOperation": PATH_ADD, "iSCSINames": add_iqns}) diff --git a/simplyblock_core/storage_arrays/adapters/huawei_dorado.py b/simplyblock_core/storage_arrays/adapters/huawei_dorado.py index 9c97615739..543f22933d 100644 --- a/simplyblock_core/storage_arrays/adapters/huawei_dorado.py +++ b/simplyblock_core/storage_arrays/adapters/huawei_dorado.py @@ -49,36 +49,64 @@ NVMe hosts are not supported by this adapter (iSCSI and FC only). +Checked against vendor source +----------------------------- +Huawei CSI ``eSDK_K8S_Plugin`` (``storage/constants.go``, +``storage/oceanstorage/base/client*.go``, ``attacher/attacher.go``, +``oceanstor/client/client_lun.go``, ``csi/backend/plugin/oceanstor.go``) and +the OpenStack Cinder Huawei driver (``huawei/rest_client.py``, +``constants.py``, ``huawei_utils.py``): + +* Login/logout, ``deviceid``/``iBaseToken``, the envelope and every error + code listed above (``WrongPasswordErrorCodes``, ``AccountBeenLocked``, + ``NeedReLogin`` on -401 / 1077949069, the per-object constants). +* ``CAPACITY`` in 512-byte sectors (CSI sends ``ceil(bytes / 512)`` with no + further rounding; the 1 MiB rounding here is merely conservative); + ``ALLOCTYPE`` 1 = thin; ``PUT /lun/expand {ID, CAPACITY}``. +* Names: 31 characters (CSI ``maxLunNameLength``, Cinder + ``MAX_NAME_LENGTH``; V6 may allow more). +* ``filter=NAME::&range=[0-100]`` with a literal ``::`` (CSI uses the + first row as the exact match; only a value is ever escaped). The adapter + used to URL-encode the whole ``filter`` parameter (``NAME%3A%3A...``); it + now builds the query like the vendor drivers. +* Initiators: ``GET /iscsi_initiator/`` / ``/fc_initiator/`` + (1077948996 = unknown), ``ISFREE``/``PARENTID`` ownership as in CSI's + attacher, ``POST /iscsi_initiator {ID[, TYPE]}`` (1077948997 = exists), + bind with ``PUT .../ {PARENTTYPE: 21, PARENTID}``; unbind with ``PUT + /iscsi_initiator/remove_iscsi_from_host`` / ``/fc_initiator/ + remove_fc_from_host {TYPE, ID}``; list a host's with ``?PARENTTYPE=21& + PARENTID=`` -- Cinder re-checks ``PARENTID`` on each row and so does this + adapter now (an ignored filter would otherwise unbind every initiator on + the array). +* ``POST /host {NAME, OPERATIONSYSTEM: 0}``; host group / LUN group / + mapping view calls, ``DELETE`` with a JSON body (``/lungroup/associate``, + ``/host/associate``), ``PUT /mappingview/create_associate`` / + ``remove_associate``, ``GET /lun/associate?TYPE=11&ASSOCIATEOBJTYPE=21& + ASSOCIATEOBJID=`` with ``HostLUNID`` in ``ASSOCIATEMETADATA``. +* ``GET /system/``: ``ID``, ``PRODUCTMODE``, ``PRODUCTVERSION``, + ``productModeString``, ``pointRelease``. + Verify against a real system ---------------------------- -* Size granularity: CAPACITY is in 512-byte sectors, but whether Dorado - accepts any sector count is unverified; we round sizes to 1 MiB to be safe. - Whether CAPACITY stays in 512-byte units for 4K-sector LUNs (``SECTORSIZE``). -* Name limits: LUN names and host names assumed <= 31 characters. -* ``filter=NAME::x`` is assumed to be an exact match (results are - re-filtered by exact name anyway); query values are sent URL-encoded. +* Whether ``CAPACITY`` stays in 512-byte units for 4K-sector LUNs + (``SECTORSIZE``). * Creating an FC initiator object for a WWPN that has never logged in - (``POST /fc_initiator {ID, TYPE: "223"}``). -* Unbinding initiators: ``PUT /iscsi_initiator/remove_iscsi_from_host`` and - ``PUT /fc_initiator/remove_fc_from_host`` with ``{ID, TYPE}`` (from the - OpenStack Cinder Huawei driver), and listing them with ``?PARENTID=``. -* ``DELETE`` requests carry a JSON body (``/lungroup/associate``, - ``/host/associate``) as the CSI driver sends them. + (``POST /fc_initiator {ID, TYPE: "223"}``): neither vendor driver does it + (CSI skips unknown or offline WWPNs). * Error codes for: deleting a LUN that is still in a LUN group, deleting a host that still has initiators or group membership, binding an initiator that is bound elsewhere, and a host already in another host group (a host can belong to one host group only; the adapter checks this itself and raises :class:`ArrayConflictError`). -* ``GET /system/`` fields used for ``get_info``: ``ID`` (serial), - ``productModeString`` or ``PRODUCTMODE`` (model code), ``pointRelease`` or - ``PRODUCTVERSION``. * Some firmware may answer an expired session with HTTP 401 instead of the - -401 envelope; both are treated as an auth error. + -401 envelope (the vendor drivers only read the envelope); both are + treated as an auth error. """ from __future__ import annotations import json from typing import Any +from urllib.parse import quote from ..base import ( ArrayAdapter, @@ -206,8 +234,16 @@ def _list(self, path: str, params: dict[str, Any] | None = None) -> list[dict[st data = self._call("GET", path, params=params) return [d for d in data if isinstance(d, dict)] if isinstance(data, list) else [] + @staticmethod + def _name_query(path: str, name: str) -> str: + """``?filter=NAME::&range=[0-100]`` with the ``::`` + separator and the range brackets left literal, as Huawei's CSI driver + and Cinder driver send them; only the name itself is percent-encoded.""" + return f"{path}?filter=NAME::{quote(name, safe='')}&range=[0-100]" + def _by_name(self, path: str, name: str) -> dict[str, Any] | None: - rows = self._list(path, {"filter": f"NAME::{name}", "range": "[0-100]"}) + rows = self._list(self._name_query(path, name)) + # The filter is treated as an exact match by the vendor drivers; check. return next((r for r in rows if r.get("NAME") == name), None) # -- info / pool ------------------------------------------------------------------ @@ -332,7 +368,13 @@ def delete_host(self, host_id: str) -> None: for collection, itype, action in ( ("iscsi_initiator", TYPE_ISCSI_INITIATOR, "remove_iscsi_from_host"), ("fc_initiator", TYPE_FC_INITIATOR, "remove_fc_from_host")): - for ini in self._list(f"/{collection}", {"PARENTID": host_id}): + # Cinder re-checks PARENTID on every row of this query; so do we, + # so that an ignored filter can never unbind another host's ports. + rows = self._list(f"/{collection}", {"PARENTTYPE": OBJ_HOST, + "PARENTID": host_id}) + for ini in rows: + if str(ini.get("PARENTID") or "") != host_id: + continue self._call("PUT", f"/{collection}/{action}", body={"ID": ini["ID"], "TYPE": itype}) self._call("DELETE", f"/host/{host_id}", missing_ok=True) diff --git a/simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py b/simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py index fe18364fb1..2e026294c9 100644 --- a/simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py +++ b/simplyblock_core/storage_arrays/adapters/ibm_flashsystem.py @@ -16,11 +16,11 @@ an :class:`ArrayRequestError`): * not found: CMMVC5753E, CMMVC5754E, CMMVC5804E, CMMVC5842E -* conflict: CMMVC6035E (name exists), CMMVC6581E (IQN/NQN already assigned - or invalid), CMMVC5867E (WWPN already assigned or invalid), CMMVC5878E - (volume already mapped to that host), CMMVC6071E (mapped to another host, - needs ``force``), CMMVC5840E (volume still mapped), CMMVC5871E (host - still has mappings) +* conflict: CMMVC6035E (name exists), CMMVC6578E (iSCSI name already + assigned or invalid), CMMVC5867E (WWPN already assigned or invalid), + CMMVC9328E (NQN already assigned), CMMVC5878E (volume already mapped to + that host), CMMVC6071E (mapped to another host, needs ``force``), + CMMVC5840E (volume still mapped), CMMVC5871E (host still has mappings) Objects: volumes are addressed by their numeric id (``Volume.id``), hosts by name (``ensure_host`` returns the host name). Note that the array reuses the @@ -33,34 +33,62 @@ allocation). Resizing grows with ``expandvdisksize`` (a delta). Deletion uses ``rmvolume``. Volume identity: ``vdisk_UID`` (32 hex) -> ``naa.``. -Initiator ownership: ``mkhost``/``addhostport`` fail with CMMVC6581E / -CMMVC5867E when a port already belongs to another host; the adapter first -reads its own host (``lshost ``, ports in the ``nodes`` list) so ports -it already owns are never re-added, and treats those codes as +Hosts: ``mkhost`` takes exactly one of ``-fcwwpn`` / ``-iscsiname`` / +``-nqn``, so a host with FC and iSCSI initiators is created with one kind and +the other added with ``addhostport``. ``mkhost`` and ``addhostport`` always +carry ``-force`` (needed for FC WWPNs that are not logged in). + +Initiator ownership: ``mkhost``/``addhostport`` fail with CMMVC6578E / +CMMVC5867E / CMMVC9328E when a port already belongs to another host; the +adapter first reads its own host (``lshost ``, ports in the ``nodes`` +list) so ports it already owns are never re-added, and treats those codes as :class:`ArrayConflictError`. +Checked against vendor source +----------------------------- +IBM Ansible collection ``ibm.storage_virtualize`` (``module_utils/ +ibm_svc_utils.py``, ``modules/ibm_svc_host.py``, ``ibm_svc_manage_volume.py``, +``ibm_svc_vol_map.py``), IBM block CSI driver (``controllers/array_action/ +array_mediator_svc.py``) and the OpenStack Cinder Storwize driver +(``storwize_svc/storwize_const.py``, ``storwize_svc_common.py``): + +* REST root ``https://:7443/rest``, ``POST /rest/auth`` with + ``X-Auth-Username``/``X-Auth-Password`` -> ``{"token"}``, then + ``X-Auth-Token``; positional args in the path, flags as a JSON object. +* A failed command is HTTP 500 whose body is the CLI text with the + ``CMMVCnnnnE`` code (Ansible ``svc_obj_info`` reads the code from it). +* Codes: CMMVC5753E / CMMVC5754E / CMMVC5804E / CMMVC5842E (not found / + not mapped), CMMVC6035E (exists), CMMVC6071E (already mapped to a host), + CMMVC5867E / CMMVC6578E / CMMVC9328E (FC / iSCSI / NVMe port already + assigned or invalid). +* ``mkhost -protocol`` values ``scsi``, ``iscsi``, ``fcscsi``, ``tcpnvme``, + ``rdmanvme``, ``fcnvme``; ``mkhost``/``addhostport`` with ``-force``; + ``-fcwwpn`` ``:``-separated, ``-iscsiname``/``-nqn`` ``,``-separated, one + of them per ``mkhost``. +* ``lshost `` ports under ``nodes`` (``WWPN``, ``iscsi_name``, + ``nqn``); ``lsvdisk `` is a list (volume, then copies); + ``lsvdiskhostmap`` with one row may be a bare dict (``host_name``, + ``SCSI_id``); ``mkvdiskhostmap -force``. +* ``mkvolume`` with ``size``/``unit=b``/``thin``/``iogrp``, + ``expandvdisksize`` with a byte delta, ``rmvolume``; ``lssystem`` + ``code_level`` is ``" (build ...)"``. + Verify against a real system ---------------------------- -* REST root: ``/rest`` (as used by IBM's Ansible collection). Code level - 8.5+ documents ``/rest/v1``; set the ``rest_root`` option if ``/rest`` is - rejected. +* ``/rest/v1`` on code levels that reject ``/rest`` (no vendor source uses + it); set the ``rest_root`` option. * HTTP status on an expired token (assumed 403, 401 handled the same) and on - bad credentials at ``/rest/auth`` (assumed 403). -* Error body format: plain text with the CMMVC message (JSON bodies are also - parsed). Codes CMMVC5804E, CMMVC5840E (rmvolume of a mapped volume; newer - levels may answer with a different code), CMMVC5871E (rmhost with - mappings) and the NQN-already-assigned code (assumed CMMVC6581E as for - IQNs) are from memory of the CLI reference. -* ``mkhost -protocol`` values for NVMe hosts (``tcpnvme``, ``rdmanvme`` on - 8.6+; older levels only knew ``nvme``) -- set ``nvme_protocol_value`` if - the array rejects them. SCSI hosts are created without ``-protocol``. -* Detail views (``lsvdisk ``, ``lshost ``) may come back as a list - of dicts (object, then copies) or one dict; ``lsvdiskhostmap`` with a - single row may come back as a dict instead of a one-element list. Both - shapes are accepted. -* ``mkvolume -unit b`` with a size that is not a 512-byte multiple (we never - send one). -* ``mkhost -force`` for FC WWPNs that are not logged in (we always pass it). + bad credentials at ``/rest/auth`` (assumed 403); the Ansible collection + only special-cases 429. +* Codes CMMVC5840E (rmvolume of a mapped volume), CMMVC5871E (rmhost with + mappings) and CMMVC5878E (volume already mapped to that host) are from + memory of the CLI reference; no vendor driver handles them (Cinder and CSI + check ``lsvdiskhostmap`` first, as this adapter does). +* NVMe ``-protocol`` on levels before 8.6 (``nvme`` only?) -- set + ``nvme_protocol_value`` if the array rejects ``tcpnvme`` / ``rdmanvme``. +* Whether a SCSI host may hold both FC and iSCSI ports (``addhostport`` of the + other kind) on code levels that split ``-protocol`` into ``fcscsi`` and + ``iscsi``. SCSI hosts are created without ``-protocol`` (default ``scsi``). """ from __future__ import annotations @@ -90,15 +118,17 @@ NOT_FOUND_CODES = frozenset({"CMMVC5753E", "CMMVC5754E", "CMMVC5804E", "CMMVC5842E"}) CONFLICT_CODES = frozenset({ "CMMVC6035E", # object already exists - "CMMVC6581E", # IQN (and, we assume, NQN) already assigned or not valid + "CMMVC6578E", # iSCSI name already assigned or not valid "CMMVC5867E", # WWPN already assigned or not valid + "CMMVC9328E", # NQN already assigned "CMMVC5878E", # volume already mapped to this host "CMMVC6071E", # volume mapped to another host; needs -force "CMMVC5840E", # volume not deleted: still mapped "CMMVC5871E", # host has mappings }) #: Codes meaning "this port is owned by another host" on mkhost/addhostport. -PORT_TAKEN_CODES = frozenset({"CMMVC6581E", "CMMVC5867E"}) +PORT_TAKEN_CODES = frozenset({"CMMVC6578E", "CMMVC5867E", "CMMVC9328E"}) +_PORT_SEP = {"nqn": ",", "fcwwpn": ":", "iscsiname": ","} _NVME_PROTOCOL = {Protocol.NVME_TCP: "tcpnvme", Protocol.NVME_RDMA: "rdmanvme"} @@ -292,37 +322,34 @@ def _nvme_protocol(self, spec: HostSpec) -> str | None: def ensure_host(self, spec: HostSpec) -> str: wanted = { - "iscsiname": spec.addresses(Protocol.ISCSI), - "fcwwpn": [a.upper() for a in spec.addresses(Protocol.FC)], "nqn": spec.addresses(Protocol.NVME_TCP, Protocol.NVME_RDMA), + "fcwwpn": [a.upper() for a in spec.addresses(Protocol.FC)], + "iscsiname": spec.addresses(Protocol.ISCSI), } protocol = self._nvme_protocol(spec) host = self._host(spec.name) try: if host is None: - opts: dict[str, Any] = {"name": spec.name} - if wanted["iscsiname"]: - opts["iscsiname"] = ",".join(wanted["iscsiname"]) - if wanted["fcwwpn"]: - opts["fcwwpn"] = ":".join(wanted["fcwwpn"]) - opts["force"] = True - if wanted["nqn"]: - opts["nqn"] = ",".join(wanted["nqn"]) + # mkhost takes exactly one of -fcwwpn / -iscsiname / -nqn; the + # other port kinds are added with addhostport below. + kind = next(k for k, v in wanted.items() if v) + opts: dict[str, Any] = {"name": spec.name, + kind: _PORT_SEP[kind].join(wanted[kind]), + "force": True} if protocol: opts["protocol"] = protocol self._cmd("mkhost", **opts) - return spec.name - have = self._ports(host) + have = {k: {v.lower() for v in wanted[kind]} if k == kind else set() + for k in wanted} + else: + have = self._ports(host) for key, values in wanted.items(): missing = [v for v in values if v.lower() not in have[key] and v not in have[key]] - if not missing: - continue - sep = ":" if key == "fcwwpn" else "," - opts = {key: sep.join(missing)} - if key == "fcwwpn": - opts["force"] = True - self._cmd("addhostport", spec.name, **opts) + if missing: + port_opts: dict[str, Any] = {key: _PORT_SEP[key].join(missing), + "force": True} + self._cmd("addhostport", spec.name, **port_opts) except ArrayConflictError as e: code = _CODE_RE.search(str(e)) if code and code.group(0) in PORT_TAKEN_CODES: diff --git a/simplyblock_core/storage_arrays/adapters/infinidat.py b/simplyblock_core/storage_arrays/adapters/infinidat.py index b87c427806..d1bdef5394 100644 --- a/simplyblock_core/storage_arrays/adapters/infinidat.py +++ b/simplyblock_core/storage_arrays/adapters/infinidat.py @@ -10,11 +10,12 @@ ``error.message``. Lists are paged (``page``, ``page_size`` up to 1000; ``metadata.pages_total``) and filtered with ``field=eq:value``. -Destructive requests (delete volume/host, unmap, shrink) need -``?approved=true``; without it the array answers **403** -``APPROVAL_REQUIRED``, which must not be mistaken for an auth failure, so -the adapter always sends the flag and maps that code to -:class:`ArrayRequestError`. +Operations the array considers risky need ``?approved=true``; without it +the array answers ``APPROVAL_REQUIRED`` (or a more specific +``APPROVAL_REQUIRED_*`` code such as ``..._VOLUME_HAS_CHILDREN``). Like +infinisdk in non-interactive mode, the adapter sends the flag on *every* +non-GET request, and maps any ``APPROVAL_REQUIRED*`` code to +:class:`ArrayRequestError` (never to an auth failure, even on 403). Objects have integer ids, returned here as strings. Volumes are created in the pool named by the ``pool`` option (resolved to its id once per session), @@ -35,29 +36,62 @@ through a host *cluster* (``clustered: true``) are not listed by this adapter (it never creates them), so ``delete_volume`` on such a volume fails. -Higher-fidelity testing: Infinidat publishes ``infinisim`` (a Python -InfiniBox simulator serving this REST API) and ``infinisdk`` (the official -SDK). Running the contract suite's live tier against infinisim would check -this adapter's API reading; neither is a dependency here. - -Verify against a real system: - -* Size granularity/minimum: the adapter rounds to 1 MiB (minimum 1 MiB) - because InfiniBox may round byte sizes up to its own allocation unit; if - the array rounds to a coarser unit, raise ``size_granularity_bytes`` (a - silent round-up would make later resizes look like shrinks). -* ``serial`` format (with or without the leading ``6``). -* Error codes and statuses: ``VOLUME_NOT_FOUND``/``HOST_NOT_FOUND``/ - ``POOL_NOT_FOUND`` (404?), name conflict (409 ``*_NAME_CONFLICT``?), - port owned by another host (409 ``PORT_ALREADY_BELONGS_TO_HOST``?), - volume still mapped on delete, ``APPROVAL_REQUIRED`` on unmap/delete. -* FC port address format accepted/returned (colons or not) -- both are - normalised on read. -* That LUN numbers start at 1 and ``GET /volumes/{id}/luns`` includes - ``host_id``. -* ``GET /system`` field names ``serial_number``, ``model_name``, ``version``. -* NVMe/TCP (``NVMEOF`` port type?) is not implemented. +Simulator: infinisdk's test extra pins ``infinisim`` (and its doctests run +against ``infinisim.infinibox.Infinibox``), but infinisim is Infinidat's +internal package: it is on neither PyPI nor GitHub, so no public simulator +can back a live contract run. + +Checked against vendor source +----------------------------- +(Infinidat ``infinisdk``: ``core/api/api.py``, ``core/config.py``, +``core/field_filter.py``, ``core/object_query.py``, +``core/translators_and_types.py``, ``infinibox/scsi_serial.py``, +``infinibox/dataset.py``, ``infinibox/system_object.py``; Infinidat +``infinibox-csi-driver``: ``iboxapi/host.go``, ``volumes.go``, +``system.go``, ``storage/common/*.go``; Infinidat ``api_7_3`` Postman +collection.) + +* Envelope ``{result, error, metadata}``; ``metadata`` has ``page``, + ``page_size``, ``pages_total``, ``number_of_objects``; ``page_size=1000`` + works (infinisdk and the CSI driver use it); filters ``field=eq:value``. +* Basic auth on every request; an expired session answers 401 (infinisdk + logs in again on 401). +* Approval: codes ``APPROVAL_REQUIRED`` and + ``APPROVAL_REQUIRED_VOLUME_HAS_CHILDREN``; infinisdk adds + ``approved=true`` to every non-GET request. +* Not found is HTTP 404 (infinisdk ``is_in_system``) with codes + ``VOLUME_NOT_FOUND``, ``HOST_NOT_FOUND``, ``POOL_NOT_FOUND``, + ``LUN_NOT_FOUND``; mapping twice answers ``MAPPING_ALREADY_EXISTS``. +* ``serial``: infinisdk parses it as IEEE company id (first 6 hex digits, + ``742b0f``) + reserved + system id + 64-bit volume id, i.e. *without* the + NAA type nibble; hosts see ``naa.6742b0f...`` (CSI driver). +* ``PUT /volumes/{id} {"size"}`` sets the absolute size (infinisdk + ``resize`` sends current + delta); shrinking needs ``force_resize``. +* Ports: ``{type, address}`` with type ``FC``/``ISCSI``/``NVME`` + (case-insensitive: infinisdk sends ``fc``/``iscsi``, the CSI driver + ``FC``/``ISCSI``/``NVME``); FC WWPNs are accepted with colons (infinisdk) + and as 16 bare hex digits (CSI driver). +* ``POST /hosts/{id}/luns {volume_id[, lun]}`` -> LUN object; + ``GET /volumes/{id}/luns`` entries carry ``lun``, ``host_id``, + ``host_cluster_id``, ``clustered``, ``volume_id``. +* ``GET /system``: ``model``, ``full_model``, ``serial_number`` (an + integer), ``version``, ``name``. + +Verify against a real system +---------------------------- +* Size granularity/minimum: the adapter rounds to 1 MiB (minimum 1 MiB). + Volumes have ``num_blocks`` (512-byte blocks), and the CSI driver only + ever requests whole GB/GiB with a 1 GB minimum; if the array rounds MiB + sizes up, raise ``size_granularity_bytes`` (a silent round-up would make + later resizes look like shrinks). +* Codes/statuses the fake invents: name conflicts (``*_NAME_CONFLICT``, + 409?), a port owned by another host (``PORT_ALREADY_BELONGS_TO_HOST``, + 409?), deleting a mapped volume or a host with LUNs, and the HTTP status + of ``APPROVAL_REQUIRED`` (403?). +* The FC address format the array *returns* (normalised on read either way). +* That auto-assigned LUN numbers start at 1. * Name limit (adapter declares 63 characters). +* NVMe/TCP (port type ``NVME``, host NQN) is not implemented. """ from __future__ import annotations @@ -136,7 +170,11 @@ def login(self) -> None: def _send(self, method: str, path: str, *, params: dict[str, Any] | None = None, json: Any = None, missing_ok: bool = False) -> dict[str, Any] | None: """Return the response envelope, or ``None`` for a missing object when - ``missing_ok``; map envelope errors to :class:`ArrayError` types.""" + ``missing_ok``; map envelope errors to :class:`ArrayError` types. + Every non-GET request is pre-approved (``approved=true``), as infinisdk + does.""" + if method.upper() != "GET": + params = {**(params or {}), **_APPROVED} resp = self.session.request(method, _API + path, params=params, json=json, ok=(400, 403, 404, 409, 500)) try: @@ -152,7 +190,7 @@ def _send(self, method: str, path: str, *, params: dict[str, Any] | None = None, code = str(err.get("code") or "") msg = str(err.get("message") or "") or body_text(resp) text = f"{method} {path} -> HTTP {status}: {code} {msg}".strip() - if status == 403 and code == "APPROVAL_REQUIRED": + if code.startswith("APPROVAL_REQUIRED"): raise ArrayRequestError(text, status=status) if status == 403: raise ArrayAuthError(text) @@ -208,7 +246,8 @@ def _pool(self) -> int: def get_info(self) -> ArrayInfo: system = self._call("GET", "/system") or {} - return ArrayInfo(vendor=self.vendor, model=str(system.get("model_name") or "InfiniBox"), + model = system.get("model") or system.get("full_model") or "InfiniBox" + return ArrayInfo(vendor=self.vendor, model=str(model), version=str(system.get("version") or ""), serial=str(system.get("serial_number") or "")) diff --git a/simplyblock_core/storage_arrays/adapters/netapp_ontap.py b/simplyblock_core/storage_arrays/adapters/netapp_ontap.py index 6f9587f867..4b0ece5cea 100644 --- a/simplyblock_core/storage_arrays/adapters/netapp_ontap.py +++ b/simplyblock_core/storage_arrays/adapters/netapp_ontap.py @@ -44,36 +44,78 @@ contract does not: before adding one, the adapter looks it up and raises :class:`ArrayConflictError` if another host (igroup or host subsystem) has it. -Asynchronous calls (FlexVol create/resize/delete) answer ``202`` with a job -link; the adapter polls ``/api/cluster/jobs/{uuid}`` until ``success`` or +Asynchronous calls answer ``202`` with a job link: FlexVol create/resize/ +delete always, LUN and namespace create/modify/delete on some systems. The +adapter polls ``/api/cluster/jobs/{uuid}`` until ``success`` or ``failure``, at most ``job_timeout_s`` seconds. +A deleted FlexVol goes to ONTAP's volume recovery queue (still using space) +unless ``skip_recovery_queue`` is set, which sends ``force=true``. + +An igroup's ``protocol`` cannot be changed after creation, so an existing +``iscsi``/``fcp`` igroup (not created by this adapter) that would need the +other kind of initiator raises :class:`ArrayConflictError`. + +ASA r2 (``disaggregated`` clusters, ONTAP 9.16.1+) has no FlexVols; ``login`` +detects it and raises :class:`ArrayUnsupportedError`. + +Checked against vendor source +----------------------------- +Trident = github.com/NetApp/trident ``storage_drivers/ontap/api`` (its REST +client under ``rest/`` is generated from the ONTAP OpenAPI spec); Ansible = +``netapp.ontap`` collection; Cinder = ``volume/drivers/netapp/dataontap``. + +* FlexVol ``DELETE`` of an online volume needs no offline first (Ansible + ``na_ontap_volume.py`` ``rest_delete_volume``; Trident ``ontap_rest.go`` + ``destroyVolumeByUUID``); ``force`` bypasses the recovery queue + (``rest/client/storage/volume_delete_parameters.go``). +* ``aggregates`` is optional on FlexVol ``POST`` (``rest/models/volume.go``; + Ansible ``aggregates_rest`` omits it when no aggregate is given); ``|`` + is the OR operator in queries (Trident ``ontap_rest.go`` aggregates query). +* LUN/namespace ``POST``/``PATCH``/``DELETE`` and FlexVol + ``POST``/``PATCH``/``DELETE`` may answer ``202`` + job; igroup, lun-map, + subsystem, subsystem-map and subsystem-host calls are synchronous + (``rest/client/*/*_responses.go``; Trident ``lunCreate``). +* Error codes: ``4`` entry doesn't exist, ``1`` duplicate entry, ``5374242`` + LUN exists, ``5702832`` LUN create in progress, ``72090025`` subsystem + exists, ``5374922`` LUN map exists; a missing object addressed by UUID is + HTTP 404 (Trident ``abstraction_error.go``, ``ontap_rest.go`` + ``isRestNotFoundError``). The adapter classifies by status, code and + message text. +* igroup ``protocol`` is ``fcp``/``iscsi``/``mixed`` (default ``mixed``) and + "cannot be changed after creation of the group" (``rest/models/igroup.go``). +* ``nsid`` is 8 hex digits plus ``h`` (``"00000001h"``, + ``rest/models/nvme_subsystem_map.go``); paths + ``/protocols/nvme/subsystem-maps/{subsystem.uuid}/{namespace.uuid}``, + ``/protocols/nvme/subsystems/{subsystem.uuid}/hosts/{nqn}``, + ``/protocols/san/lun-maps/{lun.uuid}/{igroup.uuid}``, + ``/protocols/san/igroups/{igroup.uuid}/initiators`` + (``rest/client/n_v_me/nv_me_client.go``, ``s_a_n`` client). +* ``allow_delete_with_hosts`` on subsystem ``DELETE`` (Trident + ``NVMeSubsystemDelete``); namespace resize via ``PATCH space.size`` + (Trident ``NVMeNamespaceSetSize``). +* ``records`` arrays for bulk igroup-initiator and subsystem-host ``POST`` + (``rest/models/igroup_initiator.go``, ``nvme_subsystem_host.go``). +* NVMe hosts (NVMe/FC included) are subsystem ``hosts[].nqn`` only + (``rest/models/nvme_subsystem_host.go``). +* ``naa.600a0980`` + hex of the 12-character LUN serial (Ansible + ``na_ontap_lun_map.py``, ``na_ontap_rest_info.py``); ``serial_number`` + is exactly 12 characters (``rest/models/lun.go``). +* ASA r2 detection: ``GET /cluster?fields=disaggregated`` on 9.16.1+ + (Trident ``ontap_rest.go`` ``MinimumASAR2Version``; Cinder + ``client_cmode_rest_asar2.py`` uses ``/storage/storage-units``). + Verify against a real system ---------------------------- -* FlexVol ``DELETE`` of an online volume without a junction path succeeds - without first taking it offline (the adapter does not offline it). * A thin LUN/namespace may not be larger than its FlexVol (the fake enforces this; it is why resize grows the FlexVol first). -* ``aggregates`` omitted on FlexVol ``POST`` lets ONTAP place the volume - (9.8+); with several ``aggregates`` configured the adapter picks the one - with the most ``space.block_storage.available`` (needs read access to - ``/api/storage/aggregates``, which SVM-scoped accounts may lack). -* Name/duplicate errors: HTTP 409 vs 400, and the numeric ONTAP error codes - (the adapter classifies by status and message text, not by code). -* FlexVol duplicate name surfaces as a failed job rather than a synchronous - error. -* ``PATCH`` of an igroup's ``protocol`` to ``mixed`` (only needed for igroups - not created by this adapter). -* ``nsid`` in subsystem-maps is a hex string such as ``"00000001h"``; the - subsystem-map ``DELETE`` path is - ``/subsystem-maps/{subsystem.uuid}/{namespace.uuid}``. -* ``allow_delete_with_hosts`` on subsystem ``DELETE``. -* Namespace resize via ``PATCH space.size`` (grow only). +* With several ``aggregates`` configured the adapter picks the one with the + most ``space.block_storage.available``; needs read access to + ``/api/storage/aggregates``, which SVM-scoped accounts may lack. +* Whether a duplicate FlexVol name fails synchronously (``400``/``409``) or + as a failed job, and its exact message text. * Whether query values match case-insensitively (WWPNs are always written in lowercase colon form, IQNs in lowercase). -* ASA r2 systems (ONTAP 9.16+ "ASA r2") hide FlexVols; this adapter targets - the classic unified/ASA personality. -* NVMe/FC hosts are identified by NQN, not WWPN (see :meth:`_nqn`). """ from __future__ import annotations @@ -111,6 +153,11 @@ _LUN = "lun" _NS = "namespace" +#: ONTAP error codes (Trident ``storage_drivers/ontap/api/abstraction_error.go``). +_NOT_FOUND_CODES = frozenset({"4"}) +_CONFLICT_CODES = frozenset({"1", "5374242", "5702832", "72090025", "5374922"}) +_ASA_R2_MIN_VERSION = (9, 16, 1) + _LUN_FIELDS = "uuid,name,serial_number,space.size,location.logical_unit,location.volume,svm" _NS_FIELDS = "uuid,name,space.size,location.namespace,location.volume,svm" @@ -178,6 +225,8 @@ class NetAppOntapAdapter(ArrayAdapter): default=4096), OptionDoc("job_timeout_s", "How long to wait for an asynchronous ONTAP job.", default=300), + OptionDoc("skip_recovery_queue", "Delete FlexVols with force=true so they do not " + "linger (and use space) in the volume recovery queue.", default=False), ) def __init__(self, ctx): @@ -209,6 +258,7 @@ def __init__(self, ctx): raise ArrayError("NetApp ONTAP namespace_block_size must be 512 or 4096") self.job_timeout_s = float(self._int_option("job_timeout_s", 1, 86400)) self.os_type = str(self.option("os_type")) + self.skip_recovery_queue = _as_bool(self.option("skip_recovery_queue")) def _int_option(self, name: str, lo: int, hi: int) -> int: raw = self.option(name) @@ -226,6 +276,24 @@ def login(self) -> None: self.session.set_basic_auth(self.secret("username"), self.secret("password")) if not self._records("/api/svm/svms", {"name": self.svm, "fields": "uuid,name"}): raise ArrayNotFoundError(f"SVM {self.svm!r} not found on array {self.config.id}") + if self._is_disaggregated(): + raise ArrayUnsupportedError( + f"Array {self.config.id} is an ONTAP ASA r2 (disaggregated) system, which has " + "no FlexVols; the NetApp ONTAP adapter supports unified ONTAP and classic ASA") + + def _is_disaggregated(self) -> bool: + """ASA r2 check as in Trident: ``disaggregated``, asked for on 9.16.1+.""" + cluster = json_of(self._call("GET", "/api/cluster", params={"fields": "version"})) or {} + ver = cluster.get("version") or {} + try: + parsed = (int(ver["generation"]), int(ver["major"]), int(ver["minor"])) + except (KeyError, TypeError, ValueError): + return False + if parsed < _ASA_R2_MIN_VERSION: + return False + info = json_of(self._call("GET", "/api/cluster", + params={"fields": "disaggregated"})) or {} + return _as_bool(info.get("disaggregated", False)) def close(self) -> None: self.session.clear_auth() @@ -244,12 +312,14 @@ def _call(self, method: str, path: str, *, params: dict[str, Any] | None = None, def _error(method: str, path: str, body: dict[str, Any]) -> ArrayError: err = body.get("error") or {} message = str(err.get("message", "")) or "request failed" - code = err.get("code", "") + code = str(err.get("code", "")) text = f"{method} {path}: {message} (ONTAP error {code})" lowered = message.lower() - if "doesn't exist" in lowered or "does not exist" in lowered or "not found" in lowered: + if code in _NOT_FOUND_CODES or "doesn't exist" in lowered \ + or "does not exist" in lowered or "not found" in lowered: return ArrayNotFoundError(text) - if any(w in lowered for w in ("already", "duplicate", "in use", "is mapped")): + if code in _CONFLICT_CODES or any( + w in lowered for w in ("already", "duplicate", "in use", "is mapped")): return ArrayConflictError(text) return ArrayRequestError(text, status=400) @@ -434,7 +504,8 @@ def _grow_flexvol(self, vol: dict[str, Any], size_bytes: int) -> None: def _delete_flexvol(self, uuid: str) -> None: try: - self._async("DELETE", f"/api/storage/volumes/{uuid}") + self._async("DELETE", f"/api/storage/volumes/{uuid}", + params={"force": "true"} if self.skip_recovery_queue else None) except ArrayNotFoundError: pass @@ -456,15 +527,16 @@ def create_volume(self, name: str, size_bytes: int) -> Volume: kind = _LUN if self.family == "scsi" else _NS try: if kind == _LUN: - self._call("POST", "/api/storage/luns", params={"return_records": "true"}, json={ + self._async("POST", "/api/storage/luns", params={"return_records": "true"}, json={ "svm": self._svm_ref(), "name": path, "os_type": self.os_type, "space": {"size": size_bytes, "guarantee": {"requested": False}}, }) else: - self._call("POST", "/api/storage/namespaces", params={"return_records": "true"}, - json={"svm": self._svm_ref(), "name": path, "os_type": self.os_type, - "space": {"size": size_bytes, - "block_size": self.block_size}}) + self._async("POST", "/api/storage/namespaces", + params={"return_records": "true"}, + json={"svm": self._svm_ref(), "name": path, "os_type": self.os_type, + "space": {"size": size_bytes, + "block_size": self.block_size}}) except ArrayError: if created_flexvol: try: @@ -492,7 +564,7 @@ def resize_volume(self, volume_id: str, size_bytes: int) -> Volume: raise ArrayNotFoundError(f"FlexVol of volume {volume_id} not found") self._grow_flexvol(vol, self._flexvol_bytes(size_bytes)) base = "/api/storage/luns" if kind == _LUN else "/api/storage/namespaces" - self._call("PATCH", f"{base}/{volume_id}", json={"space": {"size": size_bytes}}) + self._async("PATCH", f"{base}/{volume_id}", json={"space": {"size": size_bytes}}) kind, rec = self._require(volume_id) return self._to_volume(kind, rec) @@ -500,7 +572,7 @@ def delete_volume(self, volume_id: str) -> None: kind, rec = self._require(volume_id) loc_vol = (rec.get("location") or {}).get("volume") or {} if kind == _LUN: - self._call("DELETE", f"/api/storage/luns/{volume_id}") + self._async("DELETE", f"/api/storage/luns/{volume_id}") else: smap = self._subsystem_map(volume_id) if smap is not None: @@ -508,7 +580,7 @@ def delete_volume(self, volume_id: str) -> None: self._call("DELETE", f"/api/protocols/nvme/subsystem-maps/{sub['uuid']}/{volume_id}") self._delete_subsystem(sub["uuid"]) - self._call("DELETE", f"/api/storage/namespaces/{volume_id}") + self._async("DELETE", f"/api/storage/namespaces/{volume_id}") # Only a FlexVol of our own (one per LUN) is removed; never the shared one. flex = loc_vol.get("name", "") if self._owns_flexvol(kind, rec): @@ -582,9 +654,14 @@ def _ensure_igroup(self, spec: HostSpec) -> str: needs = {"iscsi" if i.protocol == Protocol.ISCSI else "fcp" for i in spec.initiators if self._norm_initiator(self._igroup_initiator(i)) not in have} - if group.get("protocol") not in ("mixed", *needs) or len(needs) > 1: - self._call("PATCH", f"/api/protocols/san/igroups/{group['uuid']}", - json={"protocol": "mixed"}) + proto = group.get("protocol") or "mixed" + if proto != "mixed" and needs - {proto}: + # "The protocol of an initiator group cannot be changed after + # creation of the group." (ONTAP REST spec, igroup.protocol) + raise ArrayConflictError( + f"igroup {spec.name!r} in SVM {self.svm} has protocol {proto!r} and " + f"cannot take {', '.join(sorted(needs - {proto}))} initiators; ONTAP " + "does not allow changing an igroup's protocol") self._call("POST", f"/api/protocols/san/igroups/{group['uuid']}/initiators", json={"records": [{"name": m} for m in missing]}) return spec.name diff --git a/simplyblock_core/storage_arrays/adapters/pure.py b/simplyblock_core/storage_arrays/adapters/pure.py index 866d3894da..676f9c17d6 100644 --- a/simplyblock_core/storage_arrays/adapters/pure.py +++ b/simplyblock_core/storage_arrays/adapters/pure.py @@ -8,7 +8,45 @@ default) before eradication; ``eradicate=true`` removes them at once. Volume identity: SCSI ``naa.624a9370``; NVMe NGUID -``eui.0024a937``. +``eui.0024a937``. A connection's ``lun`` is set +for SCSI hosts only; NVMe hosts get ``nsid`` instead (REST 2.27+), which the +adapter reports as :attr:`Mapping.lun`. + +Checked against vendor source +----------------------------- +* Login: ``POST /api//login`` with header ``api-token``, session token + in the ``x-auth-token`` response header; ``POST .../logout`` ends it + (py-pure-client ``pypureclient/api_token_manager.py``, ``keywords.py`` + ``Headers``). +* Version discovery: ``GET /api/api_version`` -> ``{"version": [...]}`` + (py-pure-client ``pypureclient/_helpers.py`` ``get_target_versions``). +* ``POST /volumes?names=`` with body ``provisioned``; ``PATCH /volumes`` + (``ids``/``names``) with ``provisioned`` / ``destroyed``; ``DELETE + /volumes`` eradicates only destroyed volumes; ``GET /volumes`` without + ``destroyed=`` lists destroyed volumes too (py-pure-client + ``_common/apis/volumes_api_v_2_36.py``, ``models/volume_post_v_2_33.py``, + ``volume_patch_v_2_33.py``, ``volume_v_2_34.py``). +* ``GET/POST/DELETE /connections`` take ``host_names`` and ``volume_ids``; + items carry ``host.name``, ``volume.id``, ``lun`` and ``nsid`` + (``connections_api_v_2_36.py``, ``models/connection_v_2_27.py``). +* Hosts: ``iqns``/``wwns``/``nqns`` on POST, ``add_iqns``/``add_wwns``/ + ``add_nqns`` on PATCH, ``names`` on every call (``hosts_api_v_2_36.py``, + ``models/host_post_v_2_16.py``, ``host_patch_v_2_16.py``); WWNs are sent + as 16 hex digits without colons, upper case (Ansible + ``purestorage.flasharray`` ``plugins/modules/purefa_host.py``). +* ``naa.624a9370`` and ``eui.0024a937`` + (Ansible ``purefa_volume.py`` ``PURE_OUI`` / ``_create_nguid``; Cinder + ``volume/drivers/pure.py`` ``_get_wwn`` / ``_get_nguid``). +* Errors are HTTP 400 with ``{"errors": [{"message": ...}]}``; missing + objects say "does not exist", duplicates "already exists" (Cinder + ``pure.py`` ``ERR_MSG_*``; ``models/error_context_response_v_2_38.py``). + +Verify against a real system +---------------------------- +* Whether ``PATCH destroyed=true`` is refused for a volume that still has + host connections (the fake refuses; the contract unmaps first anyway). +* Whether REST 2.4..2.26 report NVMe connections in ``lun`` (``nsid`` was + added in 2.27). """ from __future__ import annotations @@ -206,8 +244,16 @@ def _connections(self, volume_id: str, host_id: str | None = None) -> list[Mappi if host_id is not None: params["host_names"] = host_id items = self._items("GET", "/connections", params=params, missing_ok=True) - return [Mapping(volume_id=i["volume"]["id"], host_id=i["host"]["name"], - lun=i.get("lun")) for i in items if i.get("host")] + return [self._mapping(i) for i in items if i.get("host")] + + @staticmethod + def _mapping(item: dict[str, Any]) -> Mapping: + # ``lun`` is set for SCSI connections only, ``nsid`` for NVMe ones. + number = item.get("lun") + if number is None: + number = item.get("nsid") + return Mapping(volume_id=item["volume"]["id"], host_id=item["host"]["name"], + lun=number) def map_volume(self, volume_id: str, host_id: str) -> Mapping: existing = self._connections(volume_id, host_id) @@ -215,8 +261,7 @@ def map_volume(self, volume_id: str, host_id: str) -> Mapping: return existing[0] items = self._items("POST", "/connections", params={"host_names": host_id, "volume_ids": volume_id}) - i = items[0] - return Mapping(volume_id=i["volume"]["id"], host_id=i["host"]["name"], lun=i.get("lun")) + return self._mapping(items[0]) def unmap_volume(self, volume_id: str, host_id: str) -> None: if not self._connections(volume_id, host_id): diff --git a/simplyblock_core/storage_arrays/testing/dell_powerflex.py b/simplyblock_core/storage_arrays/testing/dell_powerflex.py index d1266297df..4e08d97947 100644 --- a/simplyblock_core/storage_arrays/testing/dell_powerflex.py +++ b/simplyblock_core/storage_arrays/testing/dell_powerflex.py @@ -1,7 +1,11 @@ -"""Fake Dell PowerFlex REST gateway (3.6 ``/api/login`` token auth, 4.x -``/rest/auth/login`` OAuth), the subset the adapter uses. - -Quirks reproduced: the 3.x login token is a JSON *string*; "not found" is +"""Fake Dell PowerFlex REST gateway (``/api/login`` token auth on 3.6 and +4.x, 4.x ``/rest/auth/login`` OAuth), the subset the adapter uses. + +Quirks reproduced: the ``/api/login`` token is a JSON *string*, sent as +basic auth ``user:`` on 3.x and as ``Bearer `` on 4.x +(goscaleio); ``/api/version`` answers a JSON string (``"3.6"`` / ``"4.5"``) +and accepts the basic-auth token on both; 3.x answers 404 for +``/rest/auth/login``; "not found" is HTTP 500 with ``{"message", "httpStatusCode", "errorCode"}``; sizes round up to 8 GiB; ``mappedSdcInfo`` is ``null`` when unmapped; a second SDC mapping needs ``allowMultipleMappings``; SDCs exist before (and independently of) @@ -45,8 +49,8 @@ def __init__(self, username: str = "admin", password: str = "pf-pass", self.generation = generation self.username = username self.password = password - self.tokens: set[str] = set() # 3.x session tokens - self.access_tokens: set[str] = set() # 4.x bearer tokens + self.tokens: set[str] = set() # /api/login session tokens + self.access_tokens: set[str] = set() # 4.x /rest/auth/login tokens self.system_id = "4a54e5c8a1b2c3d4" self.pools = {"7d2e1c5b00000001": {"id": "7d2e1c5b00000001", "name": "sb-pool", "protectionDomainId": "e1f2a3b400000000"}} @@ -92,26 +96,31 @@ def add_volume(self, name: str | None, size_bytes: int, pool_id: str | None = No # -- auth --------------------------------------------------------------- - def _auth(self, req: Request) -> None: - if self.generation == 3: + def _auth(self, req: Request, basic_ok: bool = False) -> None: + if self.generation == 3 or basic_ok: creds = req.basic_auth() if creds and creds[1] in self.tokens: return - else: + if self.generation >= 4: header = req.headers.get("Authorization", "") - if header.startswith("Bearer ") and header[7:] in self.access_tokens: + if header.startswith("Bearer ") and (header[7:] in self.access_tokens + or header[7:] in self.tokens): return raise _err("Unauthorized", 401) + @route("GET", "/api/version") + def api_version(self, req): + self._auth(req, basic_ok=True) + version = "3.6" if self.generation == 3 else "4.5" + return Reply(200, json.dumps(version).encode(), {"Content-Type": "application/json"}) + @route("GET", "/api/login") def api_login(self, req): - if self.generation != 3: - raise _err("Not found", 404) if req.basic_auth() != (self.username, self.password): raise _err("Unauthorized", 401) token = secrets.token_urlsafe(24) self.tokens.add(token) - # The 3.x gateway returns the token as a JSON string literal. + # The gateway returns the token as a JSON string literal. return Reply(200, json.dumps(token).encode(), {"Content-Type": "application/json"}) @route("GET", "/api/logout") @@ -119,6 +128,9 @@ def api_logout(self, req): creds = req.basic_auth() if creds: self.tokens.discard(creds[1]) + header = req.headers.get("Authorization", "") + if header.startswith("Bearer "): + self.tokens.discard(header[7:]) return Reply(200, None) @route("POST", "/rest/auth/login") diff --git a/simplyblock_core/storage_arrays/testing/dell_powermax.py b/simplyblock_core/storage_arrays/testing/dell_powermax.py index 5c3e59f406..177139b58c 100644 --- a/simplyblock_core/storage_arrays/testing/dell_powermax.py +++ b/simplyblock_core/storage_arrays/testing/dell_powermax.py @@ -1,13 +1,15 @@ """Fake Dell PowerMax Unisphere REST API 10.x (the subset the adapter uses). Quirks reproduced: basic auth on every request (401 plain-text body); -``{"message": "..."}`` error bodies, 400 for most failures and 404 for -missing objects; versioned paths (``/univmax/restapi/{ver}/...``) with +``{"message": "..."}`` error bodies, 400 for most failures, 404 for missing +objects ("Could not find volume", "... cannot be found"), 409 for "already +exists", 500 for a masking view naming a missing port group; NVMe host +initiators as ``:<32 upper-case hex host ID>``; versioned paths (``/univmax/restapi/{ver}/...``) with unknown versions refused; sizes in cylinders (1 CYL = 1920 KiB), also accepting MB/GB/TB; ``executionOption: ASYNCHRONOUS`` answered with ``202`` + job that runs on the second poll of ``/{ver}/system/job/{id}`` -(``SCHEDULED`` -> ``RUNNING`` -> ``SUCCEEDED``/``FAILED`` with the error in -``result``); volume listing as an iterator (``count``/``resultList``, no +(``SCHEDULED`` -> ``RUNNING`` -> ``SUCCEEDED`` or ``job_failure_status`` +with the error in ``result``); volume listing as an iterator (``count``/``resultList``, no ``resultList`` when empty); non-unique volume identifiers; masked storage groups cannot lose their last device; host LUN addresses in hex; port groups bound to one protocol. @@ -75,6 +77,7 @@ def __init__(self, username: str = "smc", password: str = "pmax-pass", self.jobs: dict[str, dict[str, Any]] = {} self._job_fns: dict[str, Callable[[], Reply]] = {} self.first_lun = 1 + self.job_failure_status = "FAILED" self._dev = 0x00100 # -- fake controls ------------------------------------------------------ @@ -159,7 +162,7 @@ def get_job(self, req, ver, job_id): job["status"], job["result"] = "SUCCEEDED", "Succeeded" except FakeError as e: body = e.reply.body if isinstance(e.reply.body, dict) else {} - job["status"] = "FAILED" + job["status"] = self.job_failure_status job["result"] = f"Failed : {body.get('message', e.reply.status)}" return Reply(200, self._public_job(job_id)) @@ -182,7 +185,7 @@ def get_symmetrix(self, req, ver, sym): def _vol(self, dev: str) -> dict[str, Any]: if dev not in self.volumes: - raise _err(404, f"Cannot find Volume {dev}") + raise _err(404, f"Could not find volume {dev}") return self.volumes[dev] def _vol_view(self, v: dict[str, Any]) -> dict[str, Any]: @@ -287,7 +290,7 @@ def _sg_record(name: str, srp: str, slo: str | None) -> dict[str, Any]: def _sg(self, name: str) -> dict[str, Any]: if name not in self.storage_groups: - raise _err(404, f"Cannot find Storage Group {name}") + raise _err(404, f"The requested storage group {name} cannot be found") return self.storage_groups[name] def _sg_view(self, sg: dict[str, Any]) -> dict[str, Any]: @@ -331,7 +334,7 @@ def post_sg(self, req, ver, sym): def do() -> Reply: if name in self.storage_groups: - raise _bad(f"Storage Group {name} already exists") + raise _err(409, "The requested storage group resource already exists") slo = next((p.get("sloId") for p in params if p.get("sloId") != "None"), None) sg = self._sg_record(name, srp, slo) self.storage_groups[name] = sg @@ -360,7 +363,7 @@ def put_sg(self, req, ver, sym, name): def remove() -> Reply: for d in devs: if d not in sg["volumes"]: - raise _bad(f"Volume {d} is not in Storage Group {name}") + raise _bad(f"Storage Group {name} does not contain volume {d}") if sg["maskingview"] and len(set(sg["volumes"]) - set(devs)) == 0: raise _bad(f"Cannot remove the last volume from Storage Group {name} " "because it is part of a masking view") @@ -396,7 +399,7 @@ def delete_sg(self, req, ver, sym, name): def _host(self, name: str) -> dict[str, Any]: if name not in self.hosts: - raise _err(404, f"Cannot find Host {name}") + raise _err(404, f"Host {name} cannot be found") return self.hosts[name] def _host_view(self, h: dict[str, Any]) -> dict[str, Any]: @@ -413,6 +416,8 @@ def _claim(self, host: str, initiators: list[str]) -> list[str]: proto = _proto_of(i) if proto == "fc" and not re.match(r"^[0-9a-f]{16}$", i): raise _bad(f"Invalid initiator {i}: expected a 16-digit WWN") + if proto == "nvme-tcp" and not re.match(r"^nqn\..+:[0-9A-F]{32}$", i): + raise _bad(f"Invalid initiator {i}: expected :") for other in self.hosts.values(): if i.lower() in (x.lower() for x in other["initiator"]): if other["hostId"] == host: @@ -436,7 +441,7 @@ def post_host(self, req, ver, sym): def do() -> Reply: if name in self.hosts: - raise _bad(f"Host {name} already exists") + raise _err(409, f"Host {name} already exists") claimed = self._claim(name, initiators) self.hosts[name] = {"hostId": name, "initiator": claimed, "maskingview": []} return Reply(200, self._host_view(self.hosts[name])) @@ -474,7 +479,7 @@ def delete_host(self, req, ver, sym, name): def _view(self, name: str) -> dict[str, Any]: if name not in self.views: - raise _err(404, f"Cannot find Masking View {name}") + raise _err(404, f"Masking View {name} cannot be found") return self.views[name] @route("POST", _SLO + "/maskingview") @@ -495,10 +500,10 @@ def post_view(self, req, ver, sym): def do() -> Reply: if name in self.views: - raise _bad(f"Masking View {name} already exists") + raise _err(409, "The requested masking view resource already exists") h, s = self._host(host), self._sg(sg) if pg not in self.port_groups: - raise _err(404, f"Cannot find Port Group {pg}") + raise _err(500, "Port Group on Symmetrix cannot be found") if not s["volumes"]: raise _bad(f"Storage Group {sg} is empty") protos = {_proto_of(i) for i in h["initiator"]} @@ -584,9 +589,11 @@ def contract_setup(**options: Any) -> ContractSetup: ), Protocol.NVME_TCP: ( (Initiator(protocol=Protocol.NVME_TCP, - address="nqn.2014-08.org.nvmexpress:uuid:host-a"),), + address="nqn.2014-08.org.nvmexpress:uuid:" + "8c8e1a3c-0b7e-4b8e-9d2a-0000000000a1"),), (Initiator(protocol=Protocol.NVME_TCP, - address="nqn.2014-08.org.nvmexpress:uuid:host-b"),), + address="nqn.2014-08.org.nvmexpress:uuid:" + "8c8e1a3c-0b7e-4b8e-9d2a-0000000000b2"),), ), }, create_foreign_volume=lambda name, size: fake.add_volume(name, size, "customer_sg"), diff --git a/simplyblock_core/storage_arrays/testing/dell_powerstore.py b/simplyblock_core/storage_arrays/testing/dell_powerstore.py index 0855735314..631871d018 100644 --- a/simplyblock_core/storage_arrays/testing/dell_powerstore.py +++ b/simplyblock_core/storage_arrays/testing/dell_powerstore.py @@ -3,10 +3,14 @@ Quirks reproduced: basic auth only on ``login_session``; session cookie ``auth_cookie`` plus ``DELL-EMC-TOKEN`` CSRF header required on non-GET requests (403 without it); PostgREST-style ``select``/``eq.`` filters with -400 on unknown fields; ``limit``/``offset`` paging answered with 206 and -``Content-Range``; 201 ``{"id"}`` on create, 204 on modify/delete/attach; -422 with ``{"messages": [...]}`` for semantic errors; sizes in bytes, a -multiple of 8192; ``wwn``/``nguid`` prefixed ``naa.``/``nguid.``. +400 on unknown fields and embedded selects (``host(host_initiators)``); +``limit``/``offset`` paging answered with 206 and ``Content-Range``; 201 +``{"id"}`` on create, 204 on modify/delete/attach; 422 with +``{"messages": [...]}`` for semantic errors (a duplicate volume name answers +``name_conflict_status``, 422 or 500 on real arrays); 400 "Host is not +attached to volume" for a detach of an unattached volume; sizes in bytes, a +multiple of 8192; ``wwn``/``nguid`` prefixed ``naa.``/``nguid.``; a volume +gets its NVMe ``nsid`` on its first attach to an NVMe host. """ from __future__ import annotations @@ -31,7 +35,7 @@ GRANULARITY = 8192 _FIELDS = { - "volume": {"id", "name", "description", "type", "wwn", "nguid", "size", "state", + "volume": {"id", "name", "description", "type", "wwn", "nguid", "nsid", "size", "state", "appliance_id", "creation_timestamp", "is_replication_destination", "performance_policy_id", "protection_policy_id"}, "host": {"id", "name", "description", "os_type", "host_group_id", "host_initiators", @@ -46,6 +50,11 @@ } +# (kind, embedded relation) -> foreign key on the row +_EMBEDS = {("host_volume_mapping", "host"): "host_id", + ("host_volume_mapping", "volume"): "volume_id"} + + def _err(status: int, message: str, code: str = "0xE04040010005") -> FakeError: return FakeError(status, {"messages": [{"code": code, "severity": "Error", "message_l10n": message, "arguments": []}]}) @@ -72,6 +81,8 @@ def __init__(self, username: str = "admin", password: str = "ps-pass"): self.mappings: list[dict[str, Any]] = [] self.page_limit = 2000 # array-side default/max page size self.first_lun = 1 + self.next_nsid = 1 + self.name_conflict_status = 422 # -- fake controls ------------------------------------------------------ @@ -87,7 +98,7 @@ def add_volume(self, name: str, size: int) -> str: self.volumes[vid] = { "id": vid, "name": name, "description": None, "type": "Primary", "wwn": "naa.68ccf098" + secrets.token_hex(12), - "nguid": "nguid." + secrets.token_hex(16), + "nguid": "nguid." + secrets.token_hex(16), "nsid": None, "size": size, "state": "Ready", "appliance_id": "A1", "creation_timestamp": "2026-11-20T00:00:00+00:00", "is_replication_destination": False, "performance_policy_id": "default_medium", @@ -149,19 +160,13 @@ def _query(self, req: Request, kind: str, rows: list[dict[str, Any]]) -> Reply: want = m.group(1) rows = [r for r in rows if _as_filter(r.get(key)) == want] select = req.q("select") - if select: - names = select.split(",") - bad = [n for n in names if n not in fields] - if bad: - raise _err(400, f"Invalid select field: {','.join(bad)}.", "0xE04040020009") - else: - names = ["id"] + names = self._select(kind, select) if select else ["id"] offset = int(req.q("offset", "0") or 0) limit = min(int(req.q("limit", str(self.page_limit)) or self.page_limit), self.page_limit) total = len(rows) page = rows[offset:offset + limit] - body = [{n: r.get(n) for n in names} for r in page] + body = [self._row(kind, r, names) for r in page] if offset + len(page) < total: end = offset + len(page) - 1 return Reply(206, body, {"Content-Range": f"{offset}-{end}/{total}"}) @@ -169,11 +174,46 @@ def _query(self, req: Request, kind: str, rows: list[dict[str, Any]]) -> Reply: def _project(self, req: Request, kind: str, row: dict[str, Any]) -> Reply: select = req.q("select") - names = select.split(",") if select else sorted(_FIELDS[kind]) - bad = [n for n in names if n not in _FIELDS[kind]] + names = self._select(kind, select) if select else sorted(_FIELDS[kind]) + return Reply(200, self._row(kind, row, names)) + + @staticmethod + def _select(kind: str, select: str) -> list[str]: + """Top-level select items; ``rel(a,b)`` embeds a related object.""" + names, depth, cur = [], 0, "" + for ch in select: + if ch == "," and depth == 0: + names.append(cur) + cur = "" + continue + depth += (ch == "(") - (ch == ")") + cur += ch + names.append(cur) + bad = [] + for n in names: + m = re.match(r"^(\w+)\(([\w,]+)\)$", n) + if m: + rel, sub = m.group(1), m.group(2).split(",") + if (kind, rel) not in _EMBEDS or any(f not in _FIELDS[rel] for f in sub): + bad.append(n) + elif n not in _FIELDS[kind]: + bad.append(n) if bad: raise _err(400, f"Invalid select field: {','.join(bad)}.", "0xE04040020009") - return Reply(200, {n: row.get(n) for n in names}) + return names + + def _row(self, kind: str, row: dict[str, Any], names: list[str]) -> dict[str, Any]: + out: dict[str, Any] = {} + for n in names: + m = re.match(r"^(\w+)\(([\w,]+)\)$", n) + if not m: + out[n] = row.get(n) + continue + rel, sub = m.group(1), m.group(2).split(",") + store = self.hosts if rel == "host" else self.volumes + target = store.get(row.get(_EMBEDS[(kind, rel)]) or "") + out[rel] = {f: target.get(f) for f in sub} if target else None + return out # -- system ------------------------------------------------------------------ @@ -239,7 +279,9 @@ def post_volume(self, req): raise _err(400, "Invalid value for name.", "0xE04040020009") size = self._check_size(body.get("size")) if any(v["name"] == name for v in self.volumes.values()): - raise _unprocessable(f"The name {name} is already in use by another volume.") + raise _err(self.name_conflict_status, + f"The name {name} is already in use by another volume.", + "0xE0A07001000C") vid = self.add_volume(name, size) if body.get("appliance_id"): self.volumes[vid]["appliance_id"] = body["appliance_id"] @@ -284,6 +326,11 @@ def attach(self, req, oid): lun = next(n for n in range(self.first_lun, 16384) if n not in used) elif lun in used: raise _unprocessable(f"Logical unit number {lun} is already in use on the host.") + vol = self.volumes[oid] + nvme = all(p["port_type"] == "NVMe" for p in self.hosts[host_id]["host_initiators"]) + if nvme and self.hosts[host_id]["host_initiators"] and vol["nsid"] is None: + vol["nsid"] = self.next_nsid + self.next_nsid += 1 self.mappings.append({"id": str(uuid.uuid4()), "host_id": host_id, "host_group_id": None, "volume_id": oid, "logical_unit_number": lun}) @@ -298,7 +345,7 @@ def detach(self, req, oid): self.mappings = [m for m in self.mappings if not (m["volume_id"] == oid and m["host_id"] == host_id)] if len(self.mappings) == before: - raise _unprocessable(f"Volume {oid} is not attached to host {host_id}.") + raise _err(400, "Host is not attached to volume.", "0xE0A08001000F") return Reply(204) @route("GET", API + "/host_volume_mapping") diff --git a/simplyblock_core/storage_arrays/testing/hitachi_vsp.py b/simplyblock_core/storage_arrays/testing/hitachi_vsp.py index cc09025c60..77f63c11cf 100644 --- a/simplyblock_core/storage_arrays/testing/hitachi_vsp.py +++ b/simplyblock_core/storage_arrays/testing/hitachi_vsp.py @@ -8,7 +8,18 @@ read back as ``NOT DEFINED`` after deletion, per-port host groups with default group 0, FC WWN uniqueness per port (iSCSI names may repeat across targets, as on the array), LU paths listed on the LDEV, the resource-group -lock, and ``{"errorSource", "message", "solution", "messageId"}`` errors. +lock (under ``/ConfigurationManager/v1[/{storageDeviceId}]/services``, not +``/objects``), and ``{"errorSource", "message", "solution", "messageId"}`` +errors. ``GET /ldevs`` treats ``count`` as a window of LDEV numbers from +``headLdevId`` (a window past the last LDEV number answers KART40044-E), which +is how Hitachi's Cinder driver pages through it. + +Message ids and SSB codes the adapter acts on follow Hitachi's Cinder driver +(``hbsd_rest_api.py``): KART30013-E (object does not exist), KART40050-E.. +KART40052-E (resource group locked), SSB 2E22/0001 (LDEV already defined), +2E11/2209 (no free LDEV number), B958/015A (LU path already defined), +B958/0947 (another LDEV on that LUN), B957/4184 (too many WWNs). Other SSBs +here are illustrative. """ from __future__ import annotations @@ -26,6 +37,7 @@ _ROOT = "/ConfigurationManager/v1/objects" _P = _ROOT + r"(?:/storages/(?P\d{12}))?" +_SVC = r"/ConfigurationManager/v1(?:/(?P\d{12}))?/services" _MAX_LDEV = 65279 NOT_DEFINED = "NOT DEFINED" _UNITS = {"K": 1 << 10, "M": 1 << 20, "G": 1 << 30, "T": 1 << 40} @@ -78,6 +90,8 @@ def __init__(self, username: str = "sbadmin", password: str = "vsp-secret", self.require_lock = False self.lock_owner: str | None = None self.max_count = 16384 + #: Highest LDEV number this model has (GET /ldevs windows beyond it fail). + self.last_ldev = _MAX_LDEV @staticmethod def _group(name: str, mode: str) -> dict[str, Any]: @@ -102,7 +116,7 @@ def add_ldev(self, label: str, size: int, ldev_id: int | None = None) -> str: def _free_ldev(self, start: int, end: int) -> int: lid = next((i for i in range(start, end + 1) if i not in self.ldevs), None) if lid is None: - raise _storage_err("No free LDEV number in the specified range.", "2E11", "0012") + raise _storage_err("No free LDEV number in the specified range.", "2E11", "2209") return lid def naa(self, lid: int) -> str: @@ -143,7 +157,7 @@ def post_session(self, req, sdid): def delete_session(self, req, sdid, sid): token = self._auth(req, sdid) if self.sessions.get(token) != int(sid): - raise _err(404, f"Session {sid} does not exist.", "KART40050-E") + raise _err(404, f"Session {sid} does not exist.", "KART30013-E") del self.sessions[token] if self.lock_owner == token: self.lock_owner = None @@ -164,7 +178,7 @@ def _job(self, req: Request, sdid: str | None, token: str, if not lock_request: if self.lock_owner and self.lock_owner != token: raise _err(409, "The resource group is locked by another user.", - "KART40049-E") + "KART40052-E") if self.require_lock and self.lock_owner != token: raise _storage_err("The resource group is not locked.", "2E10", "A001") jid = self.next_id() @@ -187,7 +201,7 @@ def get_job(self, req, sdid, jid): self._auth(req, sdid) job = self.jobs.get(int(jid)) if job is None: - raise _err(404, f"Job {jid} does not exist.", "KART40050-E") + raise _err(404, f"Job {jid} does not exist.", "KART30013-E") jid_i = int(jid) if jid_i in self._pending: if self._polls_left[jid_i] > 0: @@ -206,10 +220,11 @@ def get_job(self, req, sdid, jid): # -- resource-group lock ------------------------------------------------------- - @route("POST", _P + r"/services/resource-group-service/actions/(?Plock|unlock)/invoke") + @route("POST", _SVC + r"/resource-group-service/actions/(?Plock|unlock)/invoke") def rg_lock(self, req, sdid, act): token = self._auth(req, sdid) - prefix = self._prefix(sdid) + prefix = (f"/ConfigurationManager/v1/{sdid}/services" if sdid + else "/ConfigurationManager/v1/services") def run() -> list[str]: if act == "lock": @@ -219,10 +234,15 @@ def run() -> list[str]: self.lock_owner = token elif self.lock_owner == token: self.lock_owner = None - return [f"{prefix}/services/resource-group-service"] + return [f"{prefix}/resource-group-service"] return self._job(req, sdid, token, run, lock_request=True) + @route("PUT", _SVC + r"/resource-group-service/actions/(?Plock|unlock)/invoke") + def rg_lock_put(self, req, sdid, act): + # Hitachi's Cinder driver sends the lock as PUT, Ansible as POST. + return self.rg_lock(req, sdid, act) + # -- LDEVs ------------------------------------------------------------------ def _ldev_view(self, lid: int) -> dict[str, Any]: @@ -267,16 +287,21 @@ def list_ldevs(self, req, sdid): pool = req.q("poolId") if pool is not None and option != "dpVolume": raise _err(400, "poolId can be specified only with ldevOption=dpVolume.", - "KART40044-E") + "KART40041-E") if option not in (None, "defined", "dpVolume"): - raise _err(400, f"fake: ldevOption={option} not implemented", "KART40044-E") + raise _err(400, f"fake: ldevOption={option} not implemented", "KART40041-E") count = int(req.q("count", "100") or 100) if not 1 <= count <= self.max_count: raise _err(400, f"The value of count ({count}) is out of range.", "KART40042-E") head = int(req.q("headLdevId", "0") or 0) + if head > self.last_ldev: + body = _body(f"The value of headLdevId ({head}) is out of range.", "KART40044-E") + body["errorCode"] = {"errorCode": "EX_ENLDEV"} + raise FakeError(400, body) data = [self._ldev_view(lid) for lid in sorted(self.ldevs) - if lid >= head and (pool is None or self.ldevs[lid]["pool"] == int(pool))] - return Reply(200, {"data": data[:count]}) + if head <= lid < head + count + and (pool is None or self.ldevs[lid]["pool"] == int(pool))] + return Reply(200, {"data": data}) @staticmethod def _blocks(body: dict[str, Any]) -> int: @@ -305,7 +330,7 @@ def run() -> list[str]: if "ldevId" in body: lid = int(body["ldevId"]) if lid in self.ldevs: - raise _storage_err(f"LDEV {lid} is already defined.", "2E11", "0011") + raise _storage_err(f"LDEV {lid} is already defined.", "2E22", "0001") else: lid = self._free_ldev(int(body.get("startLdevId", 0)), int(body.get("endLdevId", _MAX_LDEV))) @@ -375,7 +400,7 @@ def run() -> list[str]: def _port(self, port: str | None) -> str: if port not in self.ports: - raise _err(404, f"The specified port ({port}) does not exist.", "KART40050-E") + raise _err(404, f"The specified port ({port}) does not exist.", "KART30013-E") return port # type: ignore[return-value] @route("GET", _P + r"/ports/(?P[^/]+)") @@ -411,7 +436,7 @@ def _existing_group(self, port: str | None, num: Any) -> tuple[str, int]: except (TypeError, ValueError) as e: raise _err(400, "hostGroupNumber is required.", "KART40041-E") from e if key not in self.groups: - raise _err(404, f"Host group {port},{num} does not exist.", "KART40050-E") + raise _err(404, f"Host group {port},{num} does not exist.", "KART30013-E") return key @route("GET", _P + r"/host-groups") @@ -494,7 +519,7 @@ def run() -> list[str]: if p == port and wwn in g["wwns"]: raise _storage_err( f"The WWN {wwn} is already registered in host group {p},{n}.", - "B957", "4184") + "B957", "4180") self.groups[(port, num)]["wwns"].append(wwn) return [f"{prefix}/host-wwns/{port},{num},{wwn}"] @@ -565,12 +590,12 @@ def run() -> list[str]: raise _storage_err(f"LDEV {lid} is not defined.", "2E10", "0000") if lid in g["luns"].values(): raise _storage_err(f"LDEV {lid} is already mapped in host group " - f"{port},{num}.", "B958", "0233") + f"{port},{num}.", "B958", "015A") n = int(lun) if lun is not None else next( i for i in range(2048) if i not in g["luns"]) if n in g["luns"]: raise _storage_err(f"LUN {n} is already in use in host group " - f"{port},{num}.", "B958", "0234") + f"{port},{num}.", "B958", "0947") g["luns"][n] = lid return [f"{prefix}/luns/{port},{num},{n}"] @@ -582,7 +607,7 @@ def delete_lun(self, req, sdid, port, num, lun): key = self._existing_group(port, num) prefix = self._prefix(sdid) if int(lun) not in self.groups[key]["luns"]: - raise _err(404, f"LU path {port},{num},{lun} does not exist.", "KART40050-E") + raise _err(404, f"LU path {port},{num},{lun} does not exist.", "KART30013-E") def run() -> list[str]: self.groups[key]["luns"].pop(int(lun), None) diff --git a/simplyblock_core/storage_arrays/testing/hpe_nimble.py b/simplyblock_core/storage_arrays/testing/hpe_nimble.py index f02e5a9728..350d841cc9 100644 --- a/simplyblock_core/storage_arrays/testing/hpe_nimble.py +++ b/simplyblock_core/storage_arrays/testing/hpe_nimble.py @@ -6,6 +6,16 @@ ids; sizes in MiB; volumes that must be offline before deletion and cannot be offlined or deleted while ACRs exist; FC WWPNs in colon form; and LUN 0 for every iSCSI ACR with volume-scoped targets. + +Modelled on the NimbleOS SDK and Cinder: error bodies carry a generic +``SM_http_`` message followed by the specific one (``SM_eexist``, +``SM_http_not_found``, ``SM_vol_size_decreased``, ...); offlining an offline +volume is refused with "Object is already in requested state"; and with +``async_ops`` a mutating request answers 202 + ``SM_async_job_id`` and the +job is polled on ``GET /v1/jobs/`` until ``state`` is ``done``. The +specific codes for ACR/online refusals (``SM_vol_online``, +``SM_vol_has_acl``, ``SM_vol_has_connections``, ``SM_igrp_has_acl``) are +the fake's own. """ from __future__ import annotations @@ -27,8 +37,16 @@ _NG_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9.:\-]{0,63}$") +_HTTP_CODE = {400: "SM_http_bad_request", 401: "SM_http_unauthorized", + 404: "SM_http_not_found", 409: "SM_http_conflict"} + + def _err(status: int, code: str, text: str) -> FakeError: - return FakeError(status, {"messages": [{"code": code, "severity": "error", "text": text}]}) + messages = [{"code": code, "severity": "error", "text": text}] + generic = _HTTP_CODE.get(status) + if generic and generic != code: + messages.insert(0, {"code": generic, "severity": "error", "text": text}) + return FakeError(status, {"messages": messages}) class FakeHpeNimble(FakeArray): @@ -40,6 +58,12 @@ def __init__(self, username: str = "admin", password: str = "admin", self.username = username self.password = password self.volume_scoped_targets = volume_scoped_targets + #: "METHOD kind" pairs (kind = volume, initiator_group, initiator, acr) + #: that answer 202 + an async job instead of the object. + self.async_ops: set[str] = set() + #: polls a job reports ``running`` before ``done`` + self.job_polls = 2 + self.jobs: dict[str, int] = {} # job id -> polls left self.tokens: dict[str, str] = {} # token id -> session_token self.pools = {self._new_id(0x0a): "default", self._new_id(0x0a): "fast"} self.perfpolicies = {self._new_id(0x03): "default", self._new_id(0x03): "Oracle OLTP"} @@ -110,6 +134,26 @@ def _get(self, table: dict[str, dict[str, Any]], obj_id: str, what: str) -> dict raise _err(404, "SM_http_not_found", f"Object not found: {what} {obj_id}") return table[obj_id] + def _done(self, op: str, status: int, data: Any) -> Reply: + """The reply for a completed mutation, or 202 + a job for ``async_ops``.""" + if op not in self.async_ops: + return Reply(status, {"data": data} if data is not None else {}) + jid = self._new_id(0x29) + self.jobs[jid] = self.job_polls + return Reply(202, {"messages": [{"code": "SM_async_job_id", "severity": "info", + "text": f"Job {jid} is running", + "arguments": {"job_id": jid}}]}) + + @route("GET", "/v1/jobs/" + _ID) + def get_job(self, req, obj_id): + self._auth(req) + if obj_id not in self.jobs: + raise _err(404, "SM_http_not_found", f"Object not found: job {obj_id}") + left = self.jobs[obj_id] + self.jobs[obj_id] = max(0, left - 1) + return Reply(200, {"data": {"id": obj_id, "state": "running" if left else "done", + "result": "" if left else "success"}}) + # -- auth / info ---------------------------------------------------------- @route("POST", "/v1/tokens") @@ -168,7 +212,7 @@ def post_volumes(self, req): if pp_id is not None and pp_id not in self.perfpolicies: raise _err(404, "SM_http_not_found", f"Object not found: perfpolicy {pp_id}") vid = self.add_volume(name, size << 20, pool_id, pp_id) - return Reply(201, {"data": self.volumes[vid]}) + return self._done("POST volume", 201, self.volumes[vid]) @route("GET", "/v1/volumes/detail") def volumes_detail(self, req): @@ -187,17 +231,22 @@ def put_volume(self, req, obj_id): data = self._body(req) if "size" in data: size = data["size"] - if not isinstance(size, int) or size < vol["size"]: + if not isinstance(size, int) or size < 1: + raise _err(400, "SM_invalid_arg", "Invalid argument: size (MiB)") + if size < vol["size"] and not data.get("force"): raise _err(400, "SM_vol_size_decreased", - "Volume size cannot be decreased") + "the operation will decrease the size of the volume. " + "use force option to proceed.") vol["size"] = size if "online" in data: + if bool(data["online"]) == vol["online"]: + raise _err(400, "SM_ealready", "Object is already in requested state") if not data["online"] and self._vol_acrs(obj_id) and not data.get("force"): raise _err(409, "SM_vol_has_connections", "Volume has access control records; cannot offline") vol["online"] = bool(data["online"]) vol["vol_state"] = "online" if vol["online"] else "offline" - return Reply(200, {"data": vol}) + return self._done("PUT volume", 200, vol) @route("DELETE", "/v1/volumes/" + _ID) def delete_volume(self, req, obj_id): @@ -208,7 +257,7 @@ def delete_volume(self, req, obj_id): if self._vol_acrs(obj_id): raise _err(409, "SM_vol_has_acl", "Volume has access control records") del self.volumes[obj_id] - return Reply(200, {}) + return self._done("DELETE volume", 200, None) # -- initiator groups / initiators ---------------------------------------------- @@ -250,7 +299,7 @@ def post_group(self, req): self.groups[gid] = {"id": gid, "name": name, "access_protocol": proto, "description": data.get("description", ""), "target_subnets": [], "host_type": "auto"} - return Reply(201, {"data": self._group_view(self.groups[gid])}) + return self._done("POST initiator_group", 201, self._group_view(self.groups[gid])) @route("DELETE", "/v1/initiator_groups/" + _ID) def delete_group(self, req, obj_id): @@ -262,7 +311,7 @@ def delete_group(self, req, obj_id): for iid in [i for i, v in self.initiators.items() if v["initiator_group_id"] == obj_id]: del self.initiators[iid] del self.groups[obj_id] - return Reply(200, {}) + return self._done("DELETE initiator_group", 200, None) @route("GET", "/v1/initiators/detail") def initiators_detail(self, req): @@ -295,7 +344,7 @@ def post_initiator(self, req): raise _err(409, "SM_eexist", f"Object exists: initiator {wwpn}") item.update(wwpn=wwpn, alias=data.get("alias", "")) self.initiators[item["id"]] = item - return Reply(201, {"data": item}) + return self._done("POST initiator", 201, item) # -- access control records ------------------------------------------------ @@ -330,14 +379,14 @@ def post_acr(self, req): "initiator_group_name": group["name"], "apply_to": data.get("apply_to", "both"), "lun": lun, "chap_user_id": "", "pe_id": "", "snap_id": ""} - return Reply(201, {"data": self.acrs[aid]}) + return self._done("POST acr", 201, self.acrs[aid]) @route("DELETE", "/v1/access_control_records/" + _ID) def delete_acr(self, req, obj_id): self._auth(req) self._get(self.acrs, obj_id, "access control record") del self.acrs[obj_id] - return Reply(200, {}) + return self._done("DELETE acr", 200, None) def contract_setup() -> ContractSetup: diff --git a/simplyblock_core/storage_arrays/testing/hpe_wsapi.py b/simplyblock_core/storage_arrays/testing/hpe_wsapi.py index 9b93d51bf3..9a3ffcb911 100644 --- a/simplyblock_core/storage_arrays/testing/hpe_wsapi.py +++ b/simplyblock_core/storage_arrays/testing/hpe_wsapi.py @@ -7,6 +7,17 @@ responses report ``iSCSIPaths``/``FCPaths``; VLUN creation answering 201 with an empty body and a ``Location`` header; and VLUN listings that contain the template plus one active VLUN per host path. + +Error codes and statuses follow python-3parclient's own WSAPI mock +(``test/HPE3ParMockServer_flask.py``): 5 INV_USER_PASS, 12 INV_INPUT, 15 +NON_EXISTENT_CPG, 16 EXISTENT_HOST, 17 NON_EXISTENT_HOST, 19 +NON_EXISTENT_VLUN, 22 EXISTENT_VOL, 23 NON_EXISTENT_VOL, 26 EXPORTED_VLUN, +39 INV_INPUT_WRONG_TYPE, 40 INV_INPUT_MISSING_REQUIRED, 43 +INV_INPUT_EXCEEDS_RANGE (create below 256 MiB), 44 INV_INPUT_PARAM_CONFLICT, +57 INV_INPUT_EXCEEDS_LENGTH, 73 EXISTENT_PATH, 152 INV_INPUT_VV_GROW_SIZE, +153 VV_NEW_SIZE_EXCEED_CPG_LIMIT (a *403* that is not an auth failure), plus +34 "resource in use" (409) from HPE flowkit. The session-expiry answer (401 +code 6) is a guess. """ from __future__ import annotations @@ -24,6 +35,7 @@ _A = "/api/v1" _UNIT_MIB = 256 +_MIN_MIB = 256 _MAX_NAME = 31 _QUERY_RE = re.compile(r'^"?\s*volumeName\s+EQ\s+(?P\S+?)\s*"?$') @@ -36,13 +48,16 @@ class FakeHpeWsapi(FakeArray): vendor = "hpe-wsapi" def __init__(self, username: str = "3paradm", password: str = "3pardata", - cpgs: tuple[str, ...] = ("SSD_r6",), paths_per_host: int = 2): + cpgs: tuple[str, ...] = ("SSD_r6",), paths_per_host: int = 2, + cpg_limit_mib: int | None = None): super().__init__() self.username = username self.password = password self.cpgs = set(cpgs) #: active VLUN entries listed per mapping (one per host port path) self.paths_per_host = paths_per_host + #: growth limit per volume; exceeding it answers 403 code 153 + self.cpg_limit_mib = cpg_limit_mib self.sessions: set[str] = set() self.volumes: dict[str, dict[str, Any]] = {} # name -> volume self.hosts: dict[str, dict[str, Any]] = {} # name -> host @@ -114,15 +129,15 @@ def post_volumes(self, req): self.created_bodies.append(body) name, cpg, size = body.get("name"), body.get("cpg"), body.get("sizeMiB") if not name or not cpg or not isinstance(size, int): - raise _err(400, 15, "missing required field") + raise _err(400, 40, "missing required field") if len(name) > _MAX_NAME: - raise _err(400, 21, "invalid input: string length exceeds limit") + raise _err(400, 57, "invalid input: string length exceeds limit") if cpg not in self.cpgs: raise _err(404, 15, "CPG does not exist") if name in self.volumes: raise _err(409, 22, "volume exists") - if size < 1: - raise _err(400, 40, "invalid input: number exceeds expected range") + if size < _MIN_MIB: + raise _err(400, 43, "Minimum volume size is 256 MiB") self.add_volume(name, size << 20) return Reply(201, None, {"Location": f"{_A}/volumes/{name}"}) @@ -139,10 +154,13 @@ def put_volume(self, req, name): if body.get("action") == 3: delta = body.get("sizeMiB") if not isinstance(delta, int) or delta <= 0: - raise _err(400, 40, "invalid input: number exceeds expected range") - vol["sizeMiB"] = -(-(vol["sizeMiB"] + delta) // _UNIT_MIB) * _UNIT_MIB + raise _err(400, 152, "invalid grow size") + new = -(-(vol["sizeMiB"] + delta) // _UNIT_MIB) * _UNIT_MIB + if self.cpg_limit_mib is not None and new > self.cpg_limit_mib: + raise _err(403, 153, "new volume size exceeds CPG limit") + vol["sizeMiB"] = new return Reply(200) - raise _err(400, 29, "invalid input: unsupported action") + raise _err(400, 12, "invalid input: unsupported action") @route("DELETE", _A + "/volumes/(?P[^/]+)") def delete_volume(self, req, name): @@ -166,7 +184,7 @@ def _claim(self, host: str, body: dict[str, Any]) -> None: wwns = [str(w).upper().replace(":", "") for w in body.get("FCWWNs") or []] for w in wwns: if not re.fullmatch(r"[0-9A-F]{16}", w): - raise _err(400, 32, "invalid WWN") + raise _err(400, 39, "invalid WWN") for other in self.hosts.values(): if other["name"] == host: continue @@ -189,14 +207,13 @@ def post_hosts(self, req): body = req.json() or {} name = body.get("name") if not name: - raise _err(400, 15, "missing required field") + raise _err(400, 40, "missing required field") if len(name) > _MAX_NAME: - raise _err(400, 21, "invalid input: string length exceeds limit") + raise _err(400, 57, "invalid input: string length exceeds limit") if name in self.hosts: raise _err(409, 16, "host exists") if body.get("iSCSINames") and body.get("FCWWNs"): - # Mirrors the adapter's conservative reading: one path type per request. - raise _err(400, 28, "invalid input: iSCSINames and FCWWNs are exclusive") + raise _err(400, 44, "invalid input: iSCSINames and FCWWNs are exclusive") self.hosts[name] = {"id": self.next_id(), "name": name, "persona": body.get("persona", 2), "iSCSIPaths": [], "FCPaths": [], "descriptors": {}} @@ -218,9 +235,9 @@ def put_host(self, req, name): host = self._host(name) body = req.json() or {} if body.get("pathOperation") != 1: - raise _err(400, 29, "invalid input: unsupported pathOperation") + raise _err(400, 46, "invalid input: unsupported pathOperation") if body.get("iSCSINames") and body.get("FCWWNs"): - raise _err(400, 28, "invalid input: iSCSINames and FCWWNs are exclusive") + raise _err(400, 44, "invalid input: iSCSINames and FCWWNs are exclusive") self._claim(host["name"], body) return Reply(200) @@ -272,13 +289,14 @@ def post_vluns(self, req): if host not in self.hosts: raise _err(404, 17, "host does not exist") if any(t["volumeName"] == vol and t["hostname"] == host for t in self.vluns): - raise _err(409, 29, "VLUN exists") + # The real code for a duplicate VLUN is not in vendor source. + raise _err(409, 0, "VLUN exists") used = {t["lun"] for t in self.vluns if t["hostname"] == host} start = int(body.get("lun", 0)) if body.get("autoLun"): lun = next(n for n in range(start, 16384) if n not in used) elif start in used: - raise _err(409, 29, "LUN already in use") + raise _err(409, 0, "LUN already in use") else: lun = start self.vluns.append({"volumeName": vol, "hostname": host, "lun": lun}) @@ -289,7 +307,7 @@ def delete_vlun(self, req, spec): self._auth(req) parts = unquote(spec).split(",") if len(parts) != 3 or not parts[1].isdigit(): - raise _err(400, 28, "invalid VLUN id") + raise _err(400, 12, "invalid VLUN id") vol, lun, host = parts[0], int(parts[1]), parts[2] for t in self.vluns: if t["volumeName"] == vol and t["lun"] == lun and t["hostname"] == host: diff --git a/simplyblock_core/storage_arrays/testing/huawei_dorado.py b/simplyblock_core/storage_arrays/testing/huawei_dorado.py index 408f6d22f5..8134183a5c 100644 --- a/simplyblock_core/storage_arrays/testing/huawei_dorado.py +++ b/simplyblock_core/storage_arrays/testing/huawei_dorado.py @@ -9,6 +9,8 @@ the ``iBaseToken`` header, else ``-401``; wrong password is 1077987870; * every value is a string; LUN ``CAPACITY`` is in 512-byte sectors; * initiators are free-standing objects bound to a host via ``PARENTID``; + ``ignore_parent_filter`` makes the ``?PARENTID=`` list filter a no-op; +* name lookups are ``?filter=NAME::&range=[0-100]`` (exact match); * mapping is host group + LUN group + mapping view; a host can be in one host group only; host LUN ids are assigned when a LUN becomes visible to a host and reported inside the ``ASSOCIATEMETADATA`` JSON string. @@ -97,6 +99,10 @@ def __init__(self, username: str = "admin", password: str = "Admin@storage1", self.initiators: dict[str, dict[str, dict[str, Any]]] = { "iscsi_initiator": {}, "fc_initiator": {}} self.host_lun_ids: dict[tuple[str, str], int] = {} + #: Answer ``GET /?PARENTID=`` with every initiator, as if + #: the filter were ignored (Huawei's Cinder driver re-checks PARENTID + #: on each row, so clients must not trust the filter). + self.ignore_parent_filter = False # -- fake controls ------------------------------------------------------------- @@ -304,7 +310,7 @@ def _ini_lookup(self, coll: str, iid: str) -> dict[str, Any] | None: @route("GET", _D + r"/(?Piscsi_initiator|fc_initiator)") def list_initiators(self, req, dev, coll): self._auth(req, dev) - parent = req.q("PARENTID") + parent = None if self.ignore_parent_filter else req.q("PARENTID") return _ok([i for i in self.initiators[coll].values() if parent is None or i["PARENTID"] == parent]) diff --git a/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py b/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py index 609726468c..005999ef1c 100644 --- a/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py +++ b/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py @@ -10,7 +10,10 @@ * a one-row ``lsvdiskhostmap`` comes back as a bare dict; * object ids are the lowest free integer, so a deleted volume's id is reused; * ``mkvdiskhostmap`` to a second host needs ``force`` (CMMVC6071E); -* FC WWPNs that are not logged in need ``force`` on mkhost/addhostport. +* FC WWPNs that are not logged in need ``force`` on mkhost/addhostport; +* ``mkhost`` takes only one of ``-iscsiname`` / ``-fcwwpn`` / ``-nqn``; +* a port owned by another host fails with CMMVC6578E (iSCSI), CMMVC5867E + (FC) or CMMVC9328E (NVMe). """ from __future__ import annotations @@ -31,8 +34,11 @@ "does not meet the naming rules.") E_IN_REQUEST = "CMMVC5842E The action failed because an object that was in the request does not exist." E_EXISTS = "CMMVC6035E The action failed as the object already exists." -E_IQN = ("CMMVC6581E The command has failed because the maximum number of allowed iSCSI " - "qualified names (IQNs) has been reached, or the IQN is already assigned or is not valid.") +# Codes as handled by IBM's CSI driver (array_mediator_svc.py) and Cinder +# (storwize_const.py); the message texts are approximations. +E_IQN = ("CMMVC6578E The command has failed because the iSCSI name is already assigned " + "or is not valid.") +E_NQN = "CMMVC9328E The command failed because the NQN is already assigned to a host." E_WWPN = ("CMMVC5867E The action failed because the worldwide port name is already " "assigned or is not valid.") E_ALREADY_MAPPED = ("CMMVC5878E The virtual disk (VDisk)-to-host mapping was not created " @@ -48,6 +54,7 @@ "letters, numbers, spaces, periods, dashes, and underscores. The name must " "begin with a letter or an underscore.") E_PARAM = "CMMVC5707E Required parameters are missing." +E_EXCLUSIVE = "CMMVC5731E Parameter -fcwwpn is incompatible with parameter -iscsiname." E_SHRINK = "CMMVC5709E [-size] is not a supported parameter." @@ -257,7 +264,7 @@ def _parse_ports(self, opts: dict[str, Any]) -> dict[str, list[str]]: return ports def _claim(self, host: str | None, ports: dict[str, list[str]]) -> None: - errors = {"iscsiname": E_IQN, "fcwwpn": E_WWPN, "nqn": E_IQN} + errors = {"iscsiname": E_IQN, "fcwwpn": E_WWPN, "nqn": E_NQN} for kind, values in ports.items(): for value in values: if self._owner(kind, value) is not None: # even when it is this host @@ -274,6 +281,8 @@ def _c_mkhost(self, obj, opts): protocol = opts.get("protocol", "scsi") if protocol not in ("scsi", "iscsi", "fcscsi", "tcpnvme", "rdmanvme", "fcnvme", "nvme"): raise _cli_error(E_PARAM) + if sum(1 for k in ("iscsiname", "fcwwpn", "nqn") if opts.get(k)) > 1: + raise _cli_error(E_EXCLUSIVE) ports = self._parse_ports(opts) if not any(ports.values()): raise _cli_error(E_PARAM) diff --git a/simplyblock_core/storage_arrays/testing/infinidat.py b/simplyblock_core/storage_arrays/testing/infinidat.py index 595b75cab6..2412cfa398 100644 --- a/simplyblock_core/storage_arrays/testing/infinidat.py +++ b/simplyblock_core/storage_arrays/testing/infinidat.py @@ -2,13 +2,20 @@ uses. Quirks reproduced: every reply is a ``{"result", "error", "metadata"}`` -envelope; lists are paged and filtered with ``field=eq:value``; destructive -requests without ``?approved=true`` get **403** ``APPROVAL_REQUIRED``; -object ids are integers; volume ``serial`` is the NAA-6 identifier without -its leading ``6``; LUN numbers start at 1 per host. Error codes follow the -``OBJECT_NOT_FOUND`` / ``*_CONFLICT`` style of the real API but are our -reading of it, not a verified list. For higher fidelity run the contract -suite's live tier against Infinidat's ``infinisim``. +envelope (metadata ``page``/``page_size``/``pages_total``/ +``number_of_objects``); lists are paged and filtered with +``field=eq:value``; destructive requests without ``?approved=true`` get +``APPROVAL_REQUIRED``; object ids are integers; volume ``serial`` is the +NAA-6 identifier without its leading ``6``; port types are +case-insensitive and FC addresses are accepted with or without colons; +``GET /system`` reports ``model``/``full_model``/integer ``serial_number``. + +Codes taken from Infinidat's infinisdk / CSI driver: ``VOLUME_NOT_FOUND``, +``HOST_NOT_FOUND``, ``POOL_NOT_FOUND``, ``LUN_NOT_FOUND``, +``MAPPING_ALREADY_EXISTS``, ``APPROVAL_REQUIRED``. The name-conflict, port +ownership, mapped-volume and host-with-LUNs codes, the 403 status of +``APPROVAL_REQUIRED`` and LUN numbering from 1 are the fake's own reading. +Infinidat's simulator (``infinisim``) is not publicly available. """ from __future__ import annotations @@ -59,6 +66,10 @@ def __init__(self, username: str = "admin", password: str = "ibox-pass"): self.volumes: dict[int, dict[str, Any]] = {} self.hosts: dict[int, dict[str, Any]] = {} self.luns: list[dict[str, Any]] = [] # {host_id, volume_id, lun, id} + #: require ``approved=true`` on every non-GET request, not only on + #: destructive ones (which requests need approval varies by operation + #: and version; infinisdk approves all of them) + self.approval_for_all = False self.add_pool("sb-pool") # -- fake controls ------------------------------------------------------ @@ -89,6 +100,8 @@ def add_volume(self, name: str, size: int, pool_id: int | None = None) -> int: def _auth(self, req: Request) -> None: if req.basic_auth() != (self.username, self.password): raise _err(401, "UNAUTHORIZED", "Invalid credentials") + if self.approval_for_all and req.method != "GET": + self._approved(req, f"{req.method} {req.path}") @staticmethod def _approved(req: Request, what: str) -> None: @@ -137,8 +150,9 @@ def _lun_view(self, m: dict[str, Any]) -> dict[str, Any]: @route("GET", _R + "/system") def system(self, req): self._auth(req) - return _ok({"name": "ibox-fake", "serial_number": 1250, "model_name": "F4304", - "version": "7.3.10.0", "system_type": "INFINIBOX"}) + return _ok({"name": "ibox-fake", "serial_number": 1250, "model": "F4304", + "full_model": "InfiniBox F4304", "version": "7.3.10.0", + "product_id": "INFINIBOX", "wwnn": "57:42:b0:f0:00:04:e2:00"}) @route("GET", _R + "/pools") def list_pools(self, req): @@ -240,7 +254,7 @@ def add_port(self, req, oid): h = self._host(int(oid)) body = req.json() or {} ptype = str(body.get("type", "")).upper() - if ptype not in ("ISCSI", "FC"): + if ptype not in ("ISCSI", "FC", "NVME"): raise _err(400, "BAD_REQUEST", f"Unsupported port type {ptype}") address = _port_address(ptype, str(body.get("address", ""))) for other in self.hosts.values(): diff --git a/simplyblock_core/storage_arrays/testing/netapp_ontap.py b/simplyblock_core/storage_arrays/testing/netapp_ontap.py index 2ccc0b52c8..735f31b498 100644 --- a/simplyblock_core/storage_arrays/testing/netapp_ontap.py +++ b/simplyblock_core/storage_arrays/testing/netapp_ontap.py @@ -9,11 +9,21 @@ and ``name`` are always returned), so an adapter that forgets to ask for a field does not get it. * Errors: ``{"error": {"message": ..., "code": ...}}``; ``404`` for unknown - objects, ``409`` for duplicates, ``400`` for everything else. Numeric + objects, ``409`` for duplicates, ``400`` for everything else. Codes ``4`` + (entry doesn't exist), ``5374242`` (LUN exists), ``72090025`` (subsystem + exists) and ``5374922`` (LUN map exists) are ONTAP's own, as listed in + Trident ``storage_drivers/ontap/api/abstraction_error.go``; the other codes are illustrative. * FlexVol create/resize/delete are asynchronous: ``202`` with a job link; the effect happens when the job runs, after ``job_polls`` polls of - ``/api/cluster/jobs/{uuid}``. + ``/api/cluster/jobs/{uuid}``. With ``async_san`` LUN and namespace + create/modify/delete answer the same way (the ONTAP spec allows ``202`` + for them). +* FlexVol ``DELETE`` without ``force=true`` moves the volume to + :attr:`FakeNetAppOntap.recovery_queue`. +* An igroup's ``protocol`` cannot be changed after creation. +* ``GET /api/cluster`` returns ``disaggregated`` (ASA r2) only when asked for + in ``fields``, as for SVM-scoped accounts. * Size rules: FlexVol >= 20 MiB (rounded up to 4 KiB); LUN/namespace must fit in its FlexVol (usable = size minus snapshot reserve), cannot shrink, and a namespace is a multiple of its block size. @@ -55,7 +65,7 @@ MAX_LUN = 128 << 40 -def _err(status: int, message: str, code: str = "1", target: str | None = None) -> FakeError: +def _err(status: int, message: str, code: str = "2", target: str | None = None) -> FakeError: body: dict[str, Any] = {"error": {"message": message, "code": code}} if target: body["error"]["target"] = target @@ -130,6 +140,13 @@ def __init__(self, username: str = "admin", password: str = "netapp1!", #: When True, jobs never finish. self.jobs_stuck = False self._job_failures: list[tuple[str, str]] = [] + #: When True, LUN/namespace create/modify/delete answer 202 + job. + self.async_san = False + #: FlexVols deleted without ``force=true``, by uuid. + self.recovery_queue: dict[str, dict[str, Any]] = {} + self.version = (9, 14, 1) + #: ASA r2 personality. + self.disaggregated = False # -- fake controls ------------------------------------------------------ @@ -232,6 +249,16 @@ def _usable(vol: dict[str, Any]) -> int: # -- jobs ----------------------------------------------------------------- + def _san(self, description: str, run: Callable[[], Reply]) -> Reply: + """LUN/namespace calls: synchronous, or a job when ``async_san``.""" + if not self.async_san: + return run() + + def action() -> None: + run() + + return self._job(description, action) + def _job(self, description: str, action: Callable[[], None]) -> Reply: jid = str(uuid_mod.uuid4()) failure = self._job_failures.pop(0) if self._job_failures else None @@ -270,10 +297,18 @@ def get_job(self, req, uuid): @route("GET", "/api/cluster") def cluster(self, req): self._auth(req) - return Reply(200, {"name": "fake-ontap", "uuid": "1cd8a442-86d1-11e0-ae1c-123478563412", - "version": {"full": "NetApp Release 9.14.1: Mon Jan 01 00:00:00 UTC " - "2024", "generation": 9, "major": 14, - "minor": 1}}) + gen, major, minor = self.version + rec: dict[str, Any] = { + "name": "fake-ontap", "uuid": "1cd8a442-86d1-11e0-ae1c-123478563412", + "version": {"full": f"NetApp Release {gen}.{major}.{minor}: Mon Jan 01 00:00:00 " + "UTC 2024", "generation": gen, "major": major, + "minor": minor}} + asked = {f.strip() for f in (req.q("fields") or "").split(",")} + if "disaggregated" in asked and self.version >= (9, 16, 1): + rec["disaggregated"] = self.disaggregated + if "san_optimized" in asked and self.version >= (9, 16, 1): + rec["san_optimized"] = self.disaggregated + return Reply(200, rec) @route("GET", "/api/cluster/nodes") def nodes(self, req): @@ -383,6 +418,8 @@ def delete_volume(self, req, uuid): if uuid not in self.volumes: raise _not_found() + force = req.q("force") == "true" + def delete() -> None: contained = self._luns_in(uuid) if any(o["status"]["mapped"] for o in contained): @@ -390,7 +427,9 @@ def delete() -> None: for o in contained: self.luns.pop(o["uuid"], None) self.namespaces.pop(o["uuid"], None) - del self.volumes[uuid] + vol = self.volumes.pop(uuid) + if not force: + self.recovery_queue[uuid] = vol return self._job(f"DELETE /api/storage/volumes/{uuid}", delete) @@ -427,17 +466,23 @@ def post_lun(self, req): vol = self._vol_of_path(path, svm["name"]) if not body.get("os_type"): raise _err(400, 'Field "os_type" is required.', "2") - if any(o["name"] == path for o in list(self.luns.values()) + - list(self.namespaces.values())): - raise _err(409, f'A LUN or namespace already exists at path "{path}".', "5374863") - size = int((body.get("space") or {}).get("size", 0)) - self._check_size(vol, size) - lid = self.add_lun(path, size, svm["name"]) - lun = self.luns[lid] - lun["os_type"] = body["os_type"] - requested = bool(((body.get("space") or {}).get("guarantee") or {}).get("requested")) - lun["space"]["guarantee"] = {"requested": requested, "reserved": requested} - return self._created(req, lun, f"/api/storage/luns/{lid}") + + def run() -> Reply: + if any(o["name"] == path for o in list(self.luns.values()) + + list(self.namespaces.values())): + raise _err(409, f'A LUN or namespace already exists at path "{path}".', + "5374242") + size = int((body.get("space") or {}).get("size", 0)) + self._check_size(vol, size) + lid = self.add_lun(path, size, svm["name"]) + lun = self.luns[lid] + lun["os_type"] = body["os_type"] + requested = bool(((body.get("space") or {}).get("guarantee") or {}) + .get("requested")) + lun["space"]["guarantee"] = {"requested": requested, "reserved": requested} + return self._created(req, lun, f"/api/storage/luns/{lid}") + + return self._san(f"POST /api/storage/luns {path}", run) @route("PATCH", "/api/storage/luns/" + _U) def patch_lun(self, req, uuid): @@ -446,15 +491,19 @@ def patch_lun(self, req, uuid): if lun is None: raise _not_found() size = ((req.json() or {}).get("space") or {}).get("size") - if size is not None: - size = int(size) - if size < lun["space"]["size"]: - raise _err(400, "Reducing the size of a LUN might cause permanent data loss " - "or corruption and is not supported by the LUN REST API.", - "5374922") - self._check_size(self.volumes[lun["location"]["volume"]["uuid"]], size) - lun["space"]["size"] = size - return Reply(200, {}) + + def run() -> Reply: + if size is not None: + new = int(size) + if new < lun["space"]["size"]: + raise _err(400, "Reducing the size of a LUN might cause permanent data " + "loss or corruption and is not supported by the LUN REST " + "API.", "5374923") + self._check_size(self.volumes[lun["location"]["volume"]["uuid"]], new) + lun["space"]["size"] = new + return Reply(200, {}) + + return self._san(f"PATCH /api/storage/luns/{uuid}", run) @route("DELETE", "/api/storage/luns/" + _U) def delete_lun(self, req, uuid): @@ -462,12 +511,17 @@ def delete_lun(self, req, uuid): lun = self.luns.get(uuid) if lun is None: raise _not_found() - if lun["status"]["mapped"] and req.q("allow_delete_while_mapped") != "true": - raise _err(400, "The LUN is mapped; unmap it or set allow_delete_while_mapped.", - "5374875") - self.lun_maps = [m for m in self.lun_maps if m["lun"]["uuid"] != uuid] - del self.luns[uuid] - return Reply(200, {}) + allow = req.q("allow_delete_while_mapped") == "true" + + def run() -> Reply: + if lun["status"]["mapped"] and not allow: + raise _err(400, "The LUN is mapped; unmap it or set " + "allow_delete_while_mapped.", "5374875") + self.lun_maps = [m for m in self.lun_maps if m["lun"]["uuid"] != uuid] + self.luns.pop(uuid, None) + return Reply(200, {}) + + return self._san(f"DELETE /api/storage/luns/{uuid}", run) @route("GET", "/api/storage/namespaces") def get_namespaces(self, req): @@ -483,24 +537,29 @@ def post_namespace(self, req): vol = self._vol_of_path(path, svm["name"]) if not body.get("os_type"): raise _err(400, 'Field "os_type" is required.', "2") - if any(o["name"] == path for o in list(self.luns.values()) + - list(self.namespaces.values())): - raise _err(409, f'A LUN or namespace already exists at path "{path}".', "72090019") - space = body.get("space") or {} - block = int(space.get("block_size", 4096)) - if block not in (512, 4096): - raise _err(400, "block_size must be 512 or 4096.", "72089653") - size = int(space.get("size", 0)) - self._check_size(vol, size, block) - nid = str(uuid_mod.uuid4()) - self.namespaces[nid] = { - "uuid": nid, "name": path, "svm": svm, "os_type": body["os_type"], - "space": {"size": size, "block_size": block}, - "location": {"namespace": path.rsplit("/", 1)[1], - "volume": {"name": vol["name"], "uuid": vol["uuid"]}}, - "status": {"mapped": False, "state": "online"}, - } - return self._created(req, self.namespaces[nid], f"/api/storage/namespaces/{nid}") + + def run() -> Reply: + if any(o["name"] == path for o in list(self.luns.values()) + + list(self.namespaces.values())): + raise _err(409, f'A LUN or namespace already exists at path "{path}".', + "72090019") + space = body.get("space") or {} + block = int(space.get("block_size", 4096)) + if block not in (512, 4096): + raise _err(400, "block_size must be 512 or 4096.", "72089653") + size = int(space.get("size", 0)) + self._check_size(vol, size, block) + nid = str(uuid_mod.uuid4()) + self.namespaces[nid] = { + "uuid": nid, "name": path, "svm": svm, "os_type": body["os_type"], + "space": {"size": size, "block_size": block}, + "location": {"namespace": path.rsplit("/", 1)[1], + "volume": {"name": vol["name"], "uuid": vol["uuid"]}}, + "status": {"mapped": False, "state": "online"}, + } + return self._created(req, self.namespaces[nid], f"/api/storage/namespaces/{nid}") + + return self._san(f"POST /api/storage/namespaces {path}", run) @route("PATCH", "/api/storage/namespaces/" + _U) def patch_namespace(self, req, uuid): @@ -509,14 +568,18 @@ def patch_namespace(self, req, uuid): if ns is None: raise _not_found() size = ((req.json() or {}).get("space") or {}).get("size") - if size is not None: - size = int(size) - if size < ns["space"]["size"]: - raise _err(400, "Shrinking a namespace is not supported.", "72090031") - self._check_size(self.volumes[ns["location"]["volume"]["uuid"]], size, - ns["space"]["block_size"]) - ns["space"]["size"] = size - return Reply(200, {}) + + def run() -> Reply: + if size is not None: + new = int(size) + if new < ns["space"]["size"]: + raise _err(400, "Shrinking a namespace is not supported.", "72090031") + self._check_size(self.volumes[ns["location"]["volume"]["uuid"]], new, + ns["space"]["block_size"]) + ns["space"]["size"] = new + return Reply(200, {}) + + return self._san(f"PATCH /api/storage/namespaces/{uuid}", run) @route("DELETE", "/api/storage/namespaces/" + _U) def delete_namespace(self, req, uuid): @@ -524,11 +587,17 @@ def delete_namespace(self, req, uuid): ns = self.namespaces.get(uuid) if ns is None: raise _not_found() - if ns["status"]["mapped"] and req.q("allow_delete_while_mapped") != "true": - raise _err(400, "The namespace is mapped to a subsystem.", "72090024") - self.subsystem_maps = [m for m in self.subsystem_maps if m["namespace"]["uuid"] != uuid] - del self.namespaces[uuid] - return Reply(200, {}) + allow = req.q("allow_delete_while_mapped") == "true" + + def run() -> Reply: + if ns["status"]["mapped"] and not allow: + raise _err(400, "The namespace is mapped to a subsystem.", "72090024") + self.subsystem_maps = [m for m in self.subsystem_maps + if m["namespace"]["uuid"] != uuid] + self.namespaces.pop(uuid, None) + return Reply(200, {}) + + return self._san(f"DELETE /api/storage/namespaces/{uuid}", run) # -- igroups ------------------------------------------------------------------- @@ -586,10 +655,11 @@ def patch_igroup(self, req, uuid): if group is None: raise _not_found() proto = (req.json() or {}).get("protocol") - if proto is not None: - if proto not in ("iscsi", "fcp", "mixed"): - raise _err(400, f'Invalid protocol "{proto}".', "2") - group["protocol"] = proto + if proto is not None and proto != group["protocol"]: + # ONTAP spec, igroup.protocol: "The protocol of an initiator group + # cannot be changed after creation of the group." + raise _err(400, "The protocol of an initiator group cannot be changed after " + "creation of the group.", "5374746") return Reply(200, {}) @route("POST", "/api/protocols/san/igroups/" + _U + "/initiators") @@ -697,7 +767,7 @@ def post_subsystem(self, req): raise _err(400, 'Field "os_type" is required.', "2") if any(s["name"] == name and s["svm"]["name"] == svm["name"] for s in self.subsystems.values()): - raise _err(409, f'Subsystem "{name}" already exists.', "72089709") + raise _err(409, f'Subsystem "{name}" already exists.', "72090025") hosts = [] for h in body.get("hosts") or []: self._check_nqn(h["nqn"]) @@ -747,7 +817,7 @@ def delete_subsystem(self, req, uuid): raise _not_found() if any(m["subsystem"]["uuid"] == uuid for m in self.subsystem_maps) and \ req.q("allow_delete_while_mapped") != "true": - raise _err(400, "The subsystem has mapped namespaces.", "72090025") + raise _err(400, "The subsystem has mapped namespaces.", "72090028") if sub["hosts"] and req.q("allow_delete_with_hosts") != "true": raise _err(400, "The subsystem has hosts; set allow_delete_with_hosts.", "72090026") diff --git a/simplyblock_core/storage_arrays/testing/pure.py b/simplyblock_core/storage_arrays/testing/pure.py index dd611d6bb0..fc79564cd9 100644 --- a/simplyblock_core/storage_arrays/testing/pure.py +++ b/simplyblock_core/storage_arrays/testing/pure.py @@ -1,4 +1,9 @@ -"""Fake Pure FlashArray REST 2.x (the subset the adapter uses).""" +"""Fake Pure FlashArray REST 2.x (the subset the adapter uses). + +Modelled on py-pure-client (``_common/apis/*_v_2_36.py``, ``models``): +connections of NVMe hosts carry ``nsid`` and no ``lun`` (REST 2.27+), SCSI +ones ``lun``; errors are HTTP 400 ``{"errors": [{"message", "context"}]}``. +""" from __future__ import annotations import secrets @@ -20,7 +25,7 @@ def _err(msg: str, status: int = 400) -> FakeError: class FakePureFlashArray(FakeArray): vendor = "pure-flasharray" - versions: ClassVar[list[str]] = ["1.19", "2.0", "2.4", "2.17", "2.21"] + versions: ClassVar[list[str]] = ["1.19", "2.0", "2.4", "2.17", "2.21", "2.36", "2.40"] def __init__(self, api_token: str = "pure-token"): super().__init__() @@ -28,7 +33,7 @@ def __init__(self, api_token: str = "pure-token"): self.sessions: set[str] = set() self.volumes: dict[str, dict[str, Any]] = {} # id -> volume self.hosts: dict[str, dict[str, Any]] = {} # name -> host - self.connections: list[dict[str, Any]] = [] # {host, volume_id, lun} + self.connections: list[dict[str, Any]] = [] # {host, volume_id, lun|nsid} # -- fake controls ------------------------------------------------------ @@ -202,8 +207,10 @@ def delete_hosts(self, req, ver): def _conn_item(self, c): v = self.volumes[c["volume_id"]] - return {"host": {"name": c["host"]}, "volume": {"id": v["id"], "name": v["name"]}, - "lun": c["lun"]} + item = {"host": {"name": c["host"]}, "host_group": None, + "volume": {"id": v["id"], "name": v["name"]}, "lun": None, "nsid": None} + item["nsid" if "nsid" in c else "lun"] = c.get("nsid", c.get("lun")) + return item @route("GET", _V + "/connections") def get_connections(self, req, ver): @@ -225,9 +232,10 @@ def post_connections(self, req, ver): raise _err("Host does not exist.") if any(c["volume_id"] == vid and c["host"] == host for c in self.connections): raise _err("Connection already exists.") - used = {c["lun"] for c in self.connections if c["host"] == host} - lun = next(n for n in range(1, 4096) if n not in used) - c = {"host": host, "volume_id": vid, "lun": lun} + key = "nsid" if self.hosts[host]["nqns"] else "lun" + used = {c.get(key) for c in self.connections if c["host"] == host} + c = {"host": host, "volume_id": vid, + key: next(n for n in range(1, 4096) if n not in used)} self.connections.append(c) return Reply(200, {"items": [self._conn_item(c)]}) diff --git a/tests/unit/storage_arrays/test_dell_powerflex.py b/tests/unit/storage_arrays/test_dell_powerflex.py index 504cef40ea..a8e5014ab7 100644 --- a/tests/unit/storage_arrays/test_dell_powerflex.py +++ b/tests/unit/storage_arrays/test_dell_powerflex.py @@ -45,20 +45,57 @@ def setup(request): return contract_setup(generation=request.param) -def test_auto_auth_uses_generation_specific_login(setup): +def test_auto_auth_picks_scheme_by_api_version(setup): + # goscaleio / PyPowerFlex: /api/login on both generations, /api/version + # decides basic (3.x) vs Bearer (4.x); /rest/auth/login is not probed. with _client(setup) as c: info = c.get_info() + adapter = c._adapter + assert adapter is not None + bearer = adapter.session.headers.get("Authorization", "") + basic = adapter.session.http.auth fake: FakePowerFlex = setup.fake paths = [p for _, p in fake.requests] - assert paths[0] == "/rest/auth/login" + assert paths[:2] == ["/api/login", "/api/version"] + assert "/rest/auth/login" not in paths if fake.generation == 3: - assert "/api/login" in paths and "R3_6" in info.version + assert "R3_6" in info.version and basic and not bearer else: - assert "/api/login" not in paths and "R4_5" in info.version + assert "R4_5" in info.version and bearer.startswith("Bearer ") and not basic # close() logged out assert not fake.tokens and not fake.access_tokens +def test_pinned_oauth_on_4x_uses_access_token(): + setup = contract_setup(generation=4) + with _client(setup, auth="oauth") as c: + assert "R4_5" in c.get_info().version + assert setup.fake.access_tokens and not setup.fake.tokens + assert ("POST", "/rest/auth/login") in setup.fake.requests + assert ("POST", "/rest/auth/logout") in setup.fake.requests + assert not setup.fake.access_tokens + + +def test_add_mapped_sdc_body(setup): + with _client(setup) as c: + c.get_info() + adapter = c._adapter + assert adapter is not None + sent = [] + real = adapter.session.request + + def spy(method, path, **kw): + if path.endswith("/action/addMappedSdc"): + sent.append(kw["json"]) + return real(method, path, **kw) + adapter.session.request = spy # type: ignore[method-assign] + vol = adapter.create_volume("sb-body", 8 * GIB) + host = adapter.ensure_host(HostSpec(name="sbtest-body", initiators=(_sdc(setup),))) + adapter.map_volume(vol.id, host) + assert sent == [{"sdcId": host, "allowMultipleMappings": "true", + "accessMode": "ReadWrite"}] + + def test_pinned_token_auth_skips_oauth_probe(): setup = contract_setup(generation=3) with _client(setup, auth="token") as c: diff --git a/tests/unit/storage_arrays/test_dell_powermax.py b/tests/unit/storage_arrays/test_dell_powermax.py index b3eceedfba..c9d7610d29 100644 --- a/tests/unit/storage_arrays/test_dell_powermax.py +++ b/tests/unit/storage_arrays/test_dell_powermax.py @@ -10,6 +10,7 @@ from simplyblock_core.storage_arrays.adapters.dell_powermax import ( CYLINDER_BYTES, DellPowerMaxAdapter, + powermax_nvme_initiator, ) from simplyblock_core.storage_arrays.base import ( AdapterContext, @@ -254,3 +255,77 @@ def test_basic_auth_on_every_request(setup): assert adapter.session.http.auth == ("smc", "pmax-pass") adapter.close() assert adapter.session.http.auth is None + + +def test_nvme_initiator_ids_carry_the_host_id(setup): + # csi-powermax makeNVMeInitiatorIDs: ":" + nqn = "nqn.2014-08.org.nvmexpress:uuid:c32abcdf-35f9-4800-88ad-396225c90b70" + assert powermax_nvme_initiator(nqn) == nqn + ":C32ABCDF35F9480088AD396225C90B70" + explicit = "nqn.2026-01.io.sb:node1:c32abcdf35f9480088ad396225c90b70" + assert powermax_nvme_initiator(explicit) == ( + "nqn.2026-01.io.sb:node1:C32ABCDF35F9480088AD396225C90B70") + with pytest.raises(ArrayUnsupportedError, match="host ID"): + powermax_nvme_initiator("nqn.2026-01.io.sb:node1") + + adapter = _adapter(setup) + spec = HostSpec(name="sbtest-nvme", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address=nqn),)) + host = adapter.ensure_host(spec) + assert setup.fake.hosts[host]["initiator"] == [nqn + ":C32ABCDF35F9480088AD396225C90B70"] + assert adapter.ensure_host(spec) == host # already present: no re-add + vol = adapter.create_volume("sb-nvme", GIB) + adapter.map_volume(vol.id, host) + assert setup.fake.views[f"{vol.name}_{host}"]["pg"] == "sb_pg_nvme" + with pytest.raises(ArrayUnsupportedError): + adapter.ensure_host(HostSpec(name="sbtest-nvme2", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2026-01.io.sb:x"),))) + + +def test_storage_group_payload_names_workload_none(setup): + adapter = _adapter(setup) + sent = [] + real = adapter.session.request + + def spy(method, path, **kw): + if method == "POST" and path.endswith("/storagegroup"): + sent.append(kw["json"]) + return real(method, path, **kw) + adapter.session.request = spy # type: ignore[method-assign] + adapter.create_volume("sb-wl", GIB) + (param,) = sent[0]["sloBasedStorageGroupParam"] + assert param["sloId"] == "None" and param["workloadSelection"] == "None" + + +@pytest.mark.parametrize("state", ["VALIDATE_FAILED", "UNSCHEDULED"]) +def test_any_other_job_state_is_a_failure(setup, state): + setup.fake.job_failure_status = state + adapter = _adapter(setup, async_jobs=True) + vol = adapter.create_volume("sb-jobstate", 4 * GIB) + with pytest.raises(ArrayRequestError, match=f"{state}.*larger"): + adapter.resize_volume(vol.id, 2 * GIB) + + +def test_vendor_not_found_and_conflict_wording(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-words", GIB) + host = adapter.ensure_host(_iscsi("sbtest-words")) + # "cannot be found" (404 and 500) and "Could not find volume" are missing + assert adapter._call("GET", f"{adapter._slo}/host/nobody", missing_ok=True) is None + with pytest.raises(ArrayNotFoundError, match="cannot be found"): + adapter._call("POST", f"{adapter._slo}/maskingview", json={ + "maskingViewId": "sb_mv_nopg", + "hostOrHostGroupSelection": {"useExistingHostParam": {"hostId": host}}, + "portGroupSelection": {"useExistingPortGroupParam": {"portGroupId": "nopg"}}, + "storageGroupSelection": {"useExistingStorageGroupParam": { + "storageGroupId": vol.name}}}) + with pytest.raises(ArrayNotFoundError, match="Could not find volume"): + adapter.resize_volume("0FFFF", GIB) + # removing a device its storage group lacks: "does not contain" -> missing + other = setup.fake.add_volume("sb-other", GIB) + assert adapter._call("PUT", f"{adapter._slo}/storagegroup/{vol.name}", json={ + "editStorageGroupActionParam": {"removeVolumeParam": {"volumeId": [other]}}}, + missing_ok=True) is None + # 409 "already exists" is a conflict + with pytest.raises(ArrayConflictError, match="already exists"): + adapter._call("POST", f"{adapter._slo}/host", + json={"hostId": host, "initiatorId": []}) diff --git a/tests/unit/storage_arrays/test_dell_powerstore.py b/tests/unit/storage_arrays/test_dell_powerstore.py index eac8016a3b..be12010226 100644 --- a/tests/unit/storage_arrays/test_dell_powerstore.py +++ b/tests/unit/storage_arrays/test_dell_powerstore.py @@ -210,3 +210,39 @@ def test_api_token_credential_is_refused(): with pytest.raises(ArrayError, match="username"): DellPowerStoreAdapter(AdapterContext( config, Credential({"api_token": SecretStr("t")}), session)) + + +def test_nvme_mapping_reports_volume_nsid(setup): + # gopowerstore volume_types.go: the NVMe namespace id is the volume's nsid + setup.fake.next_nsid = 42 + adapter = _adapter(setup) + vol = adapter.create_volume("sb-nsid", GIB) + nvme = adapter.ensure_host(HostSpec(name="sbtest-nsid", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:n"),))) + scsi = adapter.ensure_host(_iscsi("sbtest-nsid-scsi")) + assert adapter.map_volume(vol.id, nvme).lun == 42 + assert adapter.map_volume(vol.id, scsi).lun == 1 + assert {m.host_id: m.lun for m in adapter.list_mappings(vol.id)} == {nvme: 42, scsi: 1} + + +def test_detach_race_with_not_attached_is_idempotent(setup): + adapter = _adapter(setup) + vol = adapter.create_volume("sb-race", GIB) + host = adapter.ensure_host(_iscsi("sbtest-race")) + adapter.map_volume(vol.id, host) + # Detached elsewhere between unmap_volume's mapping check and its detach: + # the array answers 400 "Host is not attached to volume", read as missing. + listed = adapter._mappings(vol.id) + setup.fake.mappings.clear() + adapter._mappings = lambda vid: listed # type: ignore[method-assign] + adapter.unmap_volume(vol.id, host) + assert ("POST", f"/api/rest/volume/{vol.id}/detach") in setup.fake.requests + + +@pytest.mark.parametrize("status", [422, 500]) +def test_duplicate_volume_name_is_conflict(setup, status): + setup.fake.name_conflict_status = status + adapter = _adapter(setup) + adapter.create_volume("sb-dup", GIB) + with pytest.raises(ArrayConflictError, match="already in use"): + adapter.create_volume("sb-dup", GIB) diff --git a/tests/unit/storage_arrays/test_hitachi_vsp.py b/tests/unit/storage_arrays/test_hitachi_vsp.py index f020fbeb60..7da110a84d 100644 --- a/tests/unit/storage_arrays/test_hitachi_vsp.py +++ b/tests/unit/storage_arrays/test_hitachi_vsp.py @@ -154,16 +154,43 @@ def test_label_failure_deletes_the_new_ldev(env): def test_find_pages_through_ldevs(env, monkeypatch): + # count is a window of LDEV numbers: the head steps by count and the scan + # ends at the first window past the last LDEV number (KART40044-E). client, fake, _ = env monkeypatch.setattr(vsp, "PAGE_SIZE", 2) + fake.last_ldev = 9 for i in range(5): fake.add_ldev(f"other-{i}", GIB) vol_uuid = _uuid() vol = client.create_volume(vol_uuid, GIB) assert int(vol.id) == 5 + fake.requests.clear() assert client.find_volume(vol_uuid) == vol heads = [p for m, p in fake.requests if m == "GET" and p.endswith("/ldevs")] - assert len(heads) >= 3 + assert len(heads) == 6 # windows 0, 2, 4, 6, 8, then 10 is past the end + + +def test_find_scans_past_sparse_windows(env): + # An LDEV far above the first 16384 numbers is still found, although the + # first window returns (almost) nothing. + client, fake, _ = env + vol_uuid = _uuid() + name = client.volume_name(vol_uuid) + fake.add_ldev("other", GIB, ldev_id=3) + fake.add_ldev(name, GIB, ldev_id=40000) + vol = client.find_volume(vol_uuid) + assert vol is not None and vol.id == "40000" + gets = [p for m, p in fake.requests if m == "GET" and p.endswith("/ldevs")] + assert len(gets) == 4 # 0, 16384, 32768, 49152 up to LDEV 65535 + + +def test_find_first_window_error_is_not_swallowed(env): + client, fake, _ = env + client.get_info() + fake.fail_next(400, {"errorSource": "x", "message": "bad range", + "messageId": "KART40044-E"}, path=r"/ldevs$") + with pytest.raises(ArrayError, match="KART40044-E"): + client.find_volume(_uuid()) def test_duplicate_labels_conflict(env): @@ -358,3 +385,41 @@ def test_classify_error_bodies(): assert isinstance(cf, ArrayConflictError) and "SSB1=B957" in str(cf) other = vsp._classify("POST x", 503, {"message": "busy"}) assert isinstance(other, ArrayRequestError) and other.status == 503 + + +@pytest.mark.parametrize("body, kind", [ + # codes from Hitachi's Cinder driver (hbsd_rest_api.py / hbsd_rest.py) + ({"message": "x", "messageId": "KART30013-E"}, "ArrayNotFoundError"), + ({"message": "x", "messageId": "KART30000-E", "errorCode": {"errorCode": "EX_ENOOBJ"}}, + "ArrayNotFoundError"), + ({"message": "x", "messageId": "KART40050-E"}, "ArrayConflictError"), + ({"message": "x", "messageId": "KART30000-E", + "errorCode": {"SSB1": "2E22", "SSB2": "0001"}}, "ArrayConflictError"), + ({"message": "x", "messageId": "KART30000-E", + "errorCode": {"SSB1": "b958", "SSB2": "0947"}}, "ArrayConflictError"), + ({"message": "x", "messageId": "KART30000-E", + "errorCode": {"SSB1": "2E11", "SSB2": "2209"}}, "ArrayRequestError"), +]) +def test_classify_structured_codes(body, kind): + assert type(vsp._classify("POST x", 400, body)).__name__ == kind + + +def test_lock_uses_services_root(): + client, fake, _ = _client(use_lock=True) + try: + client.create_volume(_uuid(), GIB) + locks = [p for _, p in fake.requests if p.endswith("/lock/invoke")] + assert locks and all( + p == "/ConfigurationManager/v1/services/resource-group-service/actions/lock/invoke" + for p in locks) + finally: + client.close() + sdid = contract_setup().fake.storage_device_id + client, fake, _ = _client(use_lock=True, storage_device_id=sdid) + try: + client.create_volume(_uuid(), GIB) + locks = [p for _, p in fake.requests if p.endswith("/lock/invoke")] + assert locks and all(p.startswith(f"/ConfigurationManager/v1/{sdid}/services/") + for p in locks) + finally: + client.close() diff --git a/tests/unit/storage_arrays/test_hpe_nimble.py b/tests/unit/storage_arrays/test_hpe_nimble.py index aadf36ba31..04c395a954 100644 --- a/tests/unit/storage_arrays/test_hpe_nimble.py +++ b/tests/unit/storage_arrays/test_hpe_nimble.py @@ -186,3 +186,62 @@ def test_host_in_use_cannot_be_deleted(): client.map_volume(vol.id, hid) with pytest.raises(ArrayConflictError, match="SM_igrp_has_acl"): client.delete_host(hid) + + +def test_error_body_has_generic_then_specific_message(): + client, _ = _client() + client.ensure_host(_iscsi("sbdup", "iqn.2026-01.io.sb:d1")) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="SM_http_conflict: .*; SM_eexist"): + adapter._data("POST", "/v1/initiator_groups", + json={"data": {"name": "sbdup", "access_protocol": "iscsi"}}) + + +def test_retried_delete_tolerates_already_offline_volume(): + # Offlining an offline volume answers "Object is already in requested + # state" (Cinder ignores it); a delete retried after the offline step + # must not fail on it. + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + fake.volumes[vol.id]["online"] = False + assert client.delete_volume(vol.id) is True + assert vol.id not in fake.volumes + + +def test_shrink_is_refused_without_force(): + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), 2 * GIB) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayError, match="SM_vol_size_decreased"): + adapter.resize_volume(vol.id, GIB) + assert fake.volumes[vol.id]["size"] == 2048 + + +def test_async_jobs_are_awaited_and_objects_reread(): + client, fake = _client() + fake.async_ops = {"POST volume", "PUT volume", "DELETE volume", "POST initiator_group", + "POST initiator", "POST acr", "DELETE acr", "DELETE initiator_group"} + vol = client.create_volume(str(uuid.uuid4()), GIB) + assert vol.size_bytes == GIB and vol.id in fake.volumes + assert client.resize_volume(vol.id, 2 * GIB).size_bytes == 2 * GIB + hid = client.ensure_host(_iscsi("sbasync", "iqn.2026-01.io.sb:as")) + assert len(fake.initiators) == 1 + m = client.map_volume(vol.id, hid) + assert m.lun == 0 and m.host_id == hid + client.unmap_volume(vol.id, hid) + assert fake.acrs == {} + assert client.delete_volume(vol.id) is True + client.delete_host(hid) + assert fake.volumes == {} and fake.groups == {} + polls = [p for m_, p in fake.requests if m_ == "GET" and p.startswith("/v1/jobs/")] + assert len(polls) >= 3 * len(fake.jobs) and all(v == 0 for v in fake.jobs.values()) + + +def test_async_job_timeout(): + client, fake = _client(job_timeout_s=4) + fake.async_ops = {"POST volume"} + fake.job_polls = 100 + with pytest.raises(ArrayError, match="did not finish within 4"): + client.create_volume(str(uuid.uuid4()), GIB) diff --git a/tests/unit/storage_arrays/test_hpe_wsapi.py b/tests/unit/storage_arrays/test_hpe_wsapi.py index d3fef3424e..366b647d24 100644 --- a/tests/unit/storage_arrays/test_hpe_wsapi.py +++ b/tests/unit/storage_arrays/test_hpe_wsapi.py @@ -10,12 +10,17 @@ ArrayConflictError, ArrayError, ArrayNotFoundError, + ArrayRequestError, HostSpec, Initiator, Protocol, ) from simplyblock_core.storage_arrays.client import ArrayClient -from simplyblock_core.storage_arrays.testing import FakeTransport, make_config +from simplyblock_core.storage_arrays.testing import ( + FakeError, + FakeTransport, + make_config, +) from simplyblock_core.storage_arrays.testing.hpe_wsapi import contract_setup from .conftest import StaticProvider @@ -186,3 +191,53 @@ def test_host_name_limit_enforced_by_client(): with pytest.raises(ArrayError, match="exceeds 31"): client.ensure_host(_iscsi("h" * 32, "iqn.2026-01.io.sb:long")) assert not fake.requests + + +def _logins(fake): + return sum(1 for m, p in fake.requests if m == "POST" and p == "/api/v1/credentials") + + +def test_non_auth_403_fails_without_relogin(): + # 403 code 153 (VV_NEW_SIZE_EXCEED_CPG_LIMIT) is a refusal, not an + # expired session: no re-login, no credential invalidation. + client, fake = _client() + fake.cpg_limit_mib = 2048 + vol = client.create_volume(str(uuid.uuid4()), GIB) + with pytest.raises(ArrayRequestError, match="code 153") as exc: + client.resize_volume(vol.id, 4 * GIB) + assert exc.value.status == 403 + assert _logins(fake) == 1 + + +def test_expired_session_still_relogs_in(): + client, fake = _client() + client.get_info() + fake.expire_sessions() + client.get_info() + assert _logins(fake) == 2 + + +def test_create_below_256_mib_is_refused_by_array(): + client, _ = _client() + adapter = client._build() # logs in + with pytest.raises(ArrayRequestError, match="code 43"): + adapter._call("POST", "/volumes", json={"name": "tiny", "cpg": "SSD_r6", + "sizeMiB": 255}) + + +def test_code_29_is_not_a_conflict(): + # Cinder sees code 29 as HTTP 400 on a volume delete (online copy), not + # as an existing VLUN. + client, fake = _client() + vol = client.create_volume(str(uuid.uuid4()), GIB) + + def refuse(req, name): + raise FakeError(400, {"code": 29, "desc": "volume has an online copy"}) + + fake._routes = [(m, p, refuse if h == fake.delete_volume else h) + for m, p, h in fake._routes] + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayRequestError, match="code 29") as exc: + adapter.delete_volume(vol.id) + assert not isinstance(exc.value, ArrayConflictError) diff --git a/tests/unit/storage_arrays/test_huawei_dorado.py b/tests/unit/storage_arrays/test_huawei_dorado.py index 64945ff14b..51013a18c3 100644 --- a/tests/unit/storage_arrays/test_huawei_dorado.py +++ b/tests/unit/storage_arrays/test_huawei_dorado.py @@ -5,6 +5,7 @@ import pytest +from simplyblock_core.storage_arrays.adapters.huawei_dorado import HuaweiDoradoAdapter from simplyblock_core.storage_arrays.base import ( ArrayAuthError, ArrayConflictError, @@ -170,3 +171,25 @@ def test_lun_in_customer_lungroup_is_not_listed_and_blocks_delete(): with pytest.raises(ArrayRequestError): client.delete_volume(vol.id) assert client.get_volume(vol.id) is not None + + +def test_name_query_keeps_separator_literal(): + # Huawei's CSI driver and Cinder send filter=NAME::&range=[0-100] + # with a literal "::"; only the name is escaped. + q = HuaweiDoradoAdapter._name_query("/lun", "sb-a b/c") + assert q == "/lun?filter=NAME::sb-a%20b%2Fc&range=[0-100]" + client, _ = _client() + vol_uuid = str(uuid.uuid4()) + vol = client.create_volume(vol_uuid, GIB) + assert client.find_volume(vol_uuid) == vol + + +def test_delete_host_never_unbinds_foreign_initiators(): + client, fake = _client() + fake.ignore_parent_filter = True + keep = client.ensure_host(HostSpec(name="sbtest-keep", initiators=(IQN_A,))) + gone = client.ensure_host(HostSpec(name="sbtest-gone", initiators=(IQN_B,))) + client.delete_host(gone) + ini = fake.initiators["iscsi_initiator"][IQN_A.address] + assert ini["ISFREE"] == "false" and ini["PARENTID"] == keep + assert fake.initiators["iscsi_initiator"][IQN_B.address]["ISFREE"] == "true" diff --git a/tests/unit/storage_arrays/test_ibm_flashsystem.py b/tests/unit/storage_arrays/test_ibm_flashsystem.py index be9e3d5d52..0c90f09e70 100644 --- a/tests/unit/storage_arrays/test_ibm_flashsystem.py +++ b/tests/unit/storage_arrays/test_ibm_flashsystem.py @@ -185,3 +185,32 @@ def test_delete_host_with_mappings_conflicts(): client.map_volume(vol.id, host) with pytest.raises(ArrayConflictError, match="CMMVC5871E"): client.delete_host(host) + + +def test_mixed_fc_iscsi_host_uses_one_port_kind_per_mkhost(): + # mkhost takes exactly one of -fcwwpn / -iscsiname / -nqn (Ansible + # ibm_svc_host, Cinder create_host); the rest goes through addhostport. + client, fake = _client() + client.ensure_host(HostSpec(name="sbtest-mix", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:mix"), + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2d")))) + (mk,) = _cmds(fake, "mkhost") + assert mk == {"name": "sbtest-mix", "fcwwpn": "21000024FF4C1A2D", "force": True} + assert _cmds(fake, "addhostport") == [{"iscsiname": "iqn.2026-01.io.sb:mix", + "force": True}] + host = fake.hosts["sbtest-mix"] + assert host["fcwwpn"] == ["21000024FF4C1A2D"] + assert host["iscsiname"] == ["iqn.2026-01.io.sb:mix"] + + +@pytest.mark.parametrize("initiator, code", [ + (Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:shared"), "CMMVC6578E"), + (Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:s"), + "CMMVC9328E"), +]) +def test_iscsi_and_nvme_port_of_other_host_conflicts(initiator, code): + client, fake = _client() + client.ensure_host(HostSpec(name="sbtest-one", initiators=(initiator,))) + with pytest.raises(ArrayConflictError, match=f"registered to another host.*{code}"): + client.ensure_host(HostSpec(name="sbtest-two", initiators=(initiator,))) + assert "sbtest-two" not in fake.hosts diff --git a/tests/unit/storage_arrays/test_infinidat.py b/tests/unit/storage_arrays/test_infinidat.py index 6b0b3ba41f..742ac5d59b 100644 --- a/tests/unit/storage_arrays/test_infinidat.py +++ b/tests/unit/storage_arrays/test_infinidat.py @@ -82,6 +82,40 @@ def test_destructive_requests_send_approval(setup): assert not setup.fake.volumes and not setup.fake.hosts +def test_every_mutation_is_preapproved(setup): + # infinisdk sends approved=true on every non-GET request; so does the adapter. + setup.fake.approval_for_all = True + fc = Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b") + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), GIB) + vol = c.resize_volume(vol.id, 2 * GIB) + host = c.ensure_host(HostSpec(name="h1", initiators=(_iqn("a"), fc))) + c.map_volume(vol.id, host) + c.unmap_volume(vol.id, host) + assert c.delete_volume(vol.id) + c.delete_host(host) + assert not setup.fake.volumes and not setup.fake.hosts + + +@pytest.mark.parametrize("code", ["APPROVAL_REQUIRED", "APPROVAL_REQUIRED_VOLUME_HAS_CHILDREN"]) +def test_specific_approval_codes_are_not_auth_errors(setup, code): + with _client(setup) as c: + vol = c.create_volume(str(uuid.uuid4()), GIB) + setup.fake.fail_next(403, {"result": None, "error": { + "code": code, "message": "needs approval", "reasons": ["has children"]}, + "metadata": {"ready": True}}, path=r"/volumes/\d+$") + adapter = c._adapter + assert adapter is not None + with pytest.raises(ArrayRequestError, match=code): + adapter.delete_volume(vol.id) + + +def test_get_info_reads_model_field(setup): + with _client(setup) as c: + info = c.get_info() + assert info.model == "F4304" and info.serial == "1250" and info.version == "7.3.10.0" + + def test_approval_required_is_not_an_auth_error(setup): with _client(setup) as c: vol = c.create_volume(str(uuid.uuid4()), GIB) diff --git a/tests/unit/storage_arrays/test_netapp_ontap.py b/tests/unit/storage_arrays/test_netapp_ontap.py index 31d2326b76..2c17efad9f 100644 --- a/tests/unit/storage_arrays/test_netapp_ontap.py +++ b/tests/unit/storage_arrays/test_netapp_ontap.py @@ -206,7 +206,7 @@ def test_lun_create_failure_removes_new_flexvol(monkeypatch): client, fake = make() def refuse(*a, **kw): - raise FakeError(400, {"error": {"message": "LUN creation refused", "code": "1"}}) + raise FakeError(400, {"error": {"message": "LUN creation refused", "code": "2"}}) monkeypatch.setattr(fake, "_check_size", refuse) with pytest.raises(ArrayRequestError, match="refused"): @@ -245,7 +245,8 @@ def test_initiator_in_foreign_igroup_conflicts_although_ontap_allows_it(): assert "mine" not in {g["name"] for g in fake.igroups.values()} -def test_existing_iscsi_igroup_is_widened_for_fc(): +def test_existing_iscsi_igroup_cannot_take_fc(): + """ONTAP spec: an igroup's protocol cannot be changed after creation.""" client, fake = make() client.get_info() adapter = client._adapter @@ -253,11 +254,27 @@ def test_existing_iscsi_igroup_is_widened_for_fc(): adapter._call("POST", "/api/protocols/san/igroups", json={ "svm": {"name": "svm1"}, "name": "h2", "os_type": "linux", "protocol": "iscsi", "initiators": [{"name": "iqn.2026-01.io.sb:h2"}]}) - client.ensure_host(HostSpec(name="h2", initiators=( - Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:h2"), - Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:99")))) + with pytest.raises(ArrayConflictError, match="cannot take fcp"): + client.ensure_host(HostSpec(name="h2", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:h2"), + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:99")))) + assert not calls(fake, "PATCH", "/api/protocols/san/igroups/") + # more iSCSI initiators still fit + client.ensure_host(iscsi("h2", "iqn.2026-01.io.sb:h2", "iqn.2026-01.io.sb:h2b")) (group,) = fake.igroups.values() - assert group["protocol"] == "mixed" and len(group["initiators"]) == 2 + assert group["protocol"] == "iscsi" and len(group["initiators"]) == 2 + + +def test_fake_refuses_igroup_protocol_change(): + client, fake = make() + client.ensure_host(iscsi("h3", "iqn.2026-01.io.sb:h3")) + adapter = client._adapter + assert adapter is not None + (group,) = fake.igroups.values() + group["protocol"] = "iscsi" + with pytest.raises(ArrayRequestError, match="cannot be changed"): + adapter._call("PATCH", f"/api/protocols/san/igroups/{group['uuid']}", + json={"protocol": "mixed"}) def test_nvme_fc_wwpn_address_is_refused(): @@ -428,3 +445,82 @@ def test_map_to_missing_host_is_not_found(): vol = client.create_volume(str(uuid.uuid4()), GIB) with pytest.raises(ArrayNotFoundError): client.map_volume(vol.id, "ghost") + + +# -- behaviour checked against vendor source ---------------------------------------------- + + +@pytest.mark.parametrize("family", ["scsi", "nvme"]) +def test_lun_and_namespace_calls_may_be_jobs(family): + """The ONTAP spec allows 202 + job for LUN/namespace create/modify/delete.""" + client, fake = make(protocol_family=family) + fake.async_san = True + fake.job_polls = 2 + vol = client.create_volume(str(uuid.uuid4()), 4 * GIB) + assert client.get_volume(vol.id) == vol + assert client.resize_volume(vol.id, 8 * GIB).size_bytes == 8 * GIB + host = client.ensure_host(nvme("n", NQN_A) if family == "nvme" + else iscsi("s", "iqn.2026-01.io.sb:s")) + client.map_volume(vol.id, host) + client.unmap_volume(vol.id, host) + assert client.delete_volume(vol.id) + assert not fake.luns and not fake.namespaces and not fake.volumes + san = "/api/storage/luns" if family == "scsi" else "/api/storage/namespaces" + assert {fake.jobs[j]["description"].split(" ")[0] for j in fake.jobs + if san in fake.jobs[j]["description"]} == {"POST", "PATCH", "DELETE"} + + +def test_async_lun_create_failure_is_classified(): + client, fake = make() + fake.async_san = True + vol = client.create_volume(str(uuid.uuid4()), GIB) + adapter = client._adapter + assert adapter is not None + with pytest.raises(ArrayConflictError, match="5374242"): + adapter._async("POST", "/api/storage/luns", json={ + "svm": {"name": "svm1"}, "name": fake.luns[vol.id]["name"], "os_type": "linux", + "space": {"size": GIB}}) + + +def test_deleted_flexvol_goes_to_recovery_queue_by_default(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + client.delete_volume(vol.id) + assert not fake.volumes and len(fake.recovery_queue) == 1 + + +def test_skip_recovery_queue_deletes_with_force(): + client, fake = make(skip_recovery_queue=True) + vol = client.create_volume(str(uuid.uuid4()), GIB) + client.delete_volume(vol.id) + assert not fake.volumes and not fake.recovery_queue + + +def test_asa_r2_is_refused_at_login(): + client, fake = make() + fake.version = (9, 16, 1) + fake.disaggregated = True + with pytest.raises(ArrayUnsupportedError, match="ASA r2"): + client.get_info() + assert not fake.volumes + + +def test_unified_9_16_is_accepted(): + client, fake = make() + fake.version = (9, 16, 1) + assert client.get_info().version.startswith("NetApp Release 9.16.1") + assert any(m == "GET" and p == "/api/cluster" for m, p in fake.requests) + + +@pytest.mark.parametrize(("code", "message", "exc"), [ + ("4", "entry missing", ArrayNotFoundError), + ("1", "entry exists", ArrayConflictError), + ("5374242", "LUN exists.", ArrayConflictError), + ("72090025", "subsystem exists", ArrayConflictError), + ("5374922", "map exists", ArrayConflictError), + ("917627", "something else", ArrayRequestError), +]) +def test_error_codes_are_classified(code, message, exc): + err = NetAppOntapAdapter._error("POST", "/api/x", {"error": {"message": message, + "code": code}}) + assert type(err) is exc diff --git a/tests/unit/storage_arrays/test_pure.py b/tests/unit/storage_arrays/test_pure.py new file mode 100644 index 0000000000..9f5831b30c --- /dev/null +++ b/tests/unit/storage_arrays/test_pure.py @@ -0,0 +1,130 @@ +"""Pure FlashArray adapter specifics beyond the shared contract.""" +from __future__ import annotations + +import uuid + +import pytest + +from simplyblock_core.storage_arrays.adapters.pure import ( + PureFlashArrayAdapter, + pure_nguid, + pure_wwn, +) +from simplyblock_core.storage_arrays.base import ( + ArrayConflictError, + ArrayNotFoundError, + ArrayRequestError, + HostSpec, + Initiator, + Protocol, +) +from simplyblock_core.storage_arrays.client import ArrayClient +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.pure import contract_setup + +from .conftest import StaticProvider + +GIB = 1 << 30 + + +def make(**options): + setup = contract_setup() + config = setup.config.model_copy(update={"options": {**setup.config.options, **options}}) + client = ArrayClient(config, StaticProvider(setup.credential), # type: ignore[arg-type] + transport=FakeTransport(setup.fake), + session_kwargs={"sleep": lambda s: None}) + return client, setup.fake + + +def test_identity_matches_vendor_examples(): + # Sample values from Ansible purestorage.flasharray purefa_volume.py RETURN docs. + assert pure_nguid("CD6B99EF258647C5000BE684") == "eui.00cd6b99ef25864724a937c5000be684" + assert pure_wwn("361019ECACE43DB3000120A4") == "naa.624a9370361019ecace43db3000120a4" + + +def test_newest_supported_rest_version_is_used(): + client, fake = make() + client.get_info() + # The fake offers 1.19 .. 2.40; 2.40 is above what the adapter was written for. + assert ("POST", "/api/2.36/login") in fake.requests + assert all(not p.startswith("/api/2.40") for _, p in fake.requests) + + +def test_pinned_api_version(): + client, fake = make(api_version="2.21") + client.get_info() + assert ("POST", "/api/2.21/login") in fake.requests + assert ("GET", "/api/api_version") not in fake.requests + + +def test_no_usable_version(): + client, fake = make() + fake.versions = ["1.19", "2.0", "2.2"] + with pytest.raises(ArrayRequestError, match="no REST 2.x version"): + client.get_info() + + +def test_nvme_connection_reports_nsid(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(HostSpec(name="nv", initiators=( + Initiator(protocol=Protocol.NVME_TCP, address="nqn.2014-08.org.nvmexpress:uuid:nv"),))) + m = client.map_volume(vol.id, host) + (conn,) = fake.connections + assert "nsid" in conn and "lun" not in conn + assert m.lun == conn["nsid"] == 1 + assert client.list_mappings(vol.id) == [m] + + +def test_scsi_connection_reports_lun(): + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + host = client.ensure_host(HostSpec(name="sc", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:sc"),))) + assert client.map_volume(vol.id, host).lun == fake.connections[0]["lun"] == 1 + + +def test_fc_wwns_are_sent_upper_case_without_colons(): + client, fake = make() + client.ensure_host(HostSpec(name="fc", initiators=( + Initiator(protocol=Protocol.FC, address="21:00:00:24:ff:4c:1a:2b"),))) + assert fake.hosts["fc"]["wwns"] == ["21000024FF4C1A2B"] + + +def test_existing_host_gets_add_fields(): + client, fake = make() + client.ensure_host(HostSpec(name="h", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:a"),))) + client.ensure_host(HostSpec(name="h", initiators=( + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:a"), + Initiator(protocol=Protocol.ISCSI, address="iqn.2026-01.io.sb:b")))) + assert fake.hosts["h"]["iqns"] == ["iqn.2026-01.io.sb:a", "iqn.2026-01.io.sb:b"] + assert ("PATCH", "/api/2.36/hosts") in fake.requests + + +def test_eradicate_option(): + client, fake = make(eradicate=True) + vol = client.create_volume(str(uuid.uuid4()), GIB) + client.delete_volume(vol.id) + assert not fake.volumes + + client, fake = make() + vol = client.create_volume(str(uuid.uuid4()), GIB) + client.delete_volume(vol.id) + assert fake.volumes[vol.id]["destroyed"] is True + assert client.get_volume(vol.id) is None + + +@pytest.mark.parametrize(("message", "exc"), [ + ("Volume does not exist.", ArrayNotFoundError), + ("Host already exists.", ArrayConflictError), + ("Implicit truncation is not permitted.", ArrayRequestError), +]) +def test_error_messages_are_classified(message, exc): + client, fake = make() + client.get_info() + adapter = client._adapter + assert isinstance(adapter, PureFlashArrayAdapter) + fake.fail_next(400, {"errors": [{"message": message, "context": "x"}]}) + with pytest.raises(exc): + adapter._items("GET", "/volumes", params={"names": "x"}) From fdeaafeef85dc835e2c45adc5b4363bbbdb805e3 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 16:50:05 +0100 Subject: [PATCH 32/50] fix(array): registration refusal answers 400, not 422 422 is FastAPI's request-validation answer, whose detail is a list; generated clients (the operator's atlas-lib) parse it as one and lose a plain-string reason. Other v2 refusals already answer 400. Co-Authored-By: Claude Opus 5.5 --- simplyblock_web/api/v2/cluster/array.py | 2 +- tests/unit/test_array_lifecycle.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/simplyblock_web/api/v2/cluster/array.py b/simplyblock_web/api/v2/cluster/array.py index ef6afd5856..d7bcbb0fe5 100644 --- a/simplyblock_web/api/v2/cluster/array.py +++ b/simplyblock_web/api/v2/cluster/array.py @@ -56,7 +56,7 @@ def register(cluster: Cluster, parameters: ArrayParams) -> ArrayDTO: parameters.credentials, parameters.lun_size, credential_ref=parameters.credential_ref or None) except ValueError as e: - raise HTTPException(422, str(e)) + raise HTTPException(400, str(e)) return ArrayDTO(**view) diff --git a/tests/unit/test_array_lifecycle.py b/tests/unit/test_array_lifecycle.py index 3f115c6a6d..b802629cf8 100644 --- a/tests/unit/test_array_lifecycle.py +++ b/tests/unit/test_array_lifecycle.py @@ -1131,7 +1131,10 @@ def test_post_get_delete(self): dele.side_effect = ValueError("LUNs exist") self.assertEqual(c.delete(f"/clusters/{cid}/arrays/arr").status_code, 409) reg.side_effect = ValueError("bad plugin") - self.assertEqual(c.post(f"/clusters/{cid}/arrays/", json={"plugin": "x"}).status_code, 422) + r = c.post(f"/clusters/{cid}/arrays/", json={"plugin": "x"}) + # A refusal is 400 with the reason as a string; 422 is the + # request-validation answer generated clients parse as a list. + self.assertEqual((r.status_code, r.json()["detail"]), (400, "bad plugin")) class TestCli(unittest.TestCase): From 7a4a765a35279a8a47450a6c02b31ecf1a2c33cf Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 17:24:55 +0100 Subject: [PATCH 33/50] multi-attach: stale home-node error and fence/reintegrate races 2026-11-20 FTT2 bed, hard reset of sn3: the LUN takeover finished in ~1 s, then the home node's stale error_read on the dead serving node's export went through the collector's home-node branch and marked the taken-over LUN unavailable cluster-wide; on 1+0 every LVS failed I/O, SPDK blocked its ports, the monitor set the healthy nodes DOWN and the cluster suspended. - main_distr_event_collector: the home node of a multi-attach LUN served by another node is a peer of it; its errors take the peer branch. - storage_node_monitor: fenced-LUN reintegration runs in a background thread per node; its agent calls held the node's liveness checks 30 s. Resize bed, R3: the monitor unfenced a node whose SPDK was already dead (an RPC error read as "no stale stack") while the takeover's background EBS detach was still running; the detach then removed the node for good while the record listed it as a standby, and the reconcile, trusting array_attached_node_ids, never re-attached it. - lun_takeover: a node whose SPDK does not answer stays fenced; no unfence while its background array fence runs; an array fence drops the node from array_attached_node_ids so the reconcile re-presents the LUN once unfenced; the reintegration's agent call does not retry. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/controllers/lun_takeover.py | 82 +++++++++++++++++-- .../services/main_distr_event_collector.py | 8 +- .../services/storage_node_monitor.py | 21 +++-- tests/unit/test_lun_takeover.py | 61 ++++++++++++++ tests/unit/test_multi_attach_device_events.py | 70 ++++++++++++++++ 5 files changed, 227 insertions(+), 15 deletions(-) create mode 100644 tests/unit/test_multi_attach_device_events.py diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index 93d8e4e8cd..7aa8b1b793 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -67,6 +67,10 @@ NVMeDevice.STATUS_READONLY, NVMeDevice.STATUS_CANNOT_ALLOCATE) _local_locks: dict[str, threading.Lock] = {} +# (device_id, node_id) of array fences still running after a fence by +# reservation: the node is not unfenced (reintegrated) meanwhile. +_fences_in_flight: set = set() +_fences_guard = threading.Lock() _local_locks_guard = threading.Lock() @@ -339,6 +343,34 @@ def _fence(cluster, dev, failed): if not ok: raise TakeoverError(f"array did not acknowledge the fence of {host} on LUN {lun}") logger.info("LUN %s: node %s (%s) fenced at the array", lun, failed.get_id(), host) + _forget_attachment(dev.get_id(), failed.get_id()) + + +def _forget_attachment(device_id, node_id): + """The fence detached ``node_id`` from the LUN: drop it from the + device's ``array_attached_node_ids`` so the attachment reconcile, which + calls the array only for nodes it does not list, presents the LUN to it + again once the record no longer fences it. Best-effort.""" + try: + db = DBController() + dev = db.get_storage_device_by_id(device_id) + owner = db.get_storage_node_by_id(dev.node_id) + except Exception as e: # noqa: BLE001 + logger.warning("LUN fence of %s on %s: attachment record not updated: %s", + node_id, device_id, e) + return + + def _mut(n): + d = _device_on_owner(n, device_id) + if d is None or node_id not in (d.array_attached_node_ids or []): + return False + d.array_attached_node_ids = [x for x in d.array_attached_node_ids if x != node_id] + return True + try: + db.atomic_update(owner, _mut) + except Exception as e: # noqa: BLE001 + logger.warning("LUN fence of %s on %s: attachment record not updated: %s", + node_id, device_id, e) def _fence_by_reservation(cluster, dev) -> bool: @@ -351,15 +383,29 @@ def _fence_by_reservation(cluster, dev) -> bool: def _fence_in_background(cluster, dev, failed): """The array fence after a fence by reservation: off the I/O path, for good measure (the reservation already rejects the node's writes).""" + key = (dev.get_id(), failed.get_id()) + def _run(): try: _fence(cluster, dev, failed) except TakeoverError as e: logger.warning("LUN %s: array fence of %s after the reservation failed: %s; the " "attachment reconcile detaches it", dev.lun_wwn, failed.get_id(), e) + finally: + with _fences_guard: + _fences_in_flight.discard(key) + with _fences_guard: + _fences_in_flight.add(key) threading.Thread(target=_run, name=f"fence-{dev.get_id()[:8]}", daemon=True).start() +def fence_in_flight(device_id, node_id) -> bool: + """Whether an array fence of ``node_id`` on the LUN of ``device_id`` is + still running in this process.""" + with _fences_guard: + return (device_id, node_id) in _fences_in_flight + + def _wait_active(rpc, alceml, deadline): """Wait until ``alceml`` (created with a reservation key, not standby) holds the reservation and serves. Raises TakeoverError.""" @@ -764,22 +810,31 @@ def reintegrate_fenced_node(node) -> list: serving = by_id.get(dev.serving_node()) if serving is None or serving.get_id() == node_id or serving.status != StorageNode.STATUS_ONLINE: continue - try: - stale = bool(rpc.bdev_get(dev.alceml_bdev)) or bool(rpc.bdev_get(f"{dev.alceml_bdev}_PT")) - except Exception: # noqa: BLE001 - stale = False + if fence_in_flight(dev.get_id(), node_id): + # Its detach is still running: an unfence now is undone by it + # (2026-11-20 resize bed, R3: the node stayed detached for good + # while the record listed it as a standby). + logger.info("LUN reintegrate: array fence of %s on %s still running; stays fenced", + node_id, dev.lun_wwn) + continue + stale = _holds_stack(rpc, dev) + if stale is None: + # SPDK not answering (a node just lost may still read ONLINE): + # nothing proves it holds no stack. + logger.info("LUN reintegrate: SPDK of %s not answering; stays fenced from %s", + node_id, dev.lun_wwn) + continue if stale: retire_stale_local_stack(node) - try: - stale = bool(rpc.bdev_get(dev.alceml_bdev)) or bool(rpc.bdev_get(f"{dev.alceml_bdev}_PT")) - except Exception: # noqa: BLE001 - stale = False + stale = _holds_stack(rpc, dev) + if stale is None: + stale = True if stale: logger.info("LUN reintegrate: %s still holds a stack of %s; stays fenced", node_id, dev.get_id()) continue try: - result, err = node.client(timeout=15).lun_info(dev.lun_wwn) + result, err = node.client(timeout=10, retry=0, connect_retry=0).lun_info(dev.lun_wwn) except Exception as e: # noqa: BLE001 logger.info("LUN reintegrate: agent of %s not reached: %s", node_id, e) continue @@ -797,6 +852,15 @@ def reintegrate_fenced_node(node) -> list: return done +def _holds_stack(rpc, dev): + """Whether the node holds an alceml or PT bdev of ``dev``; None when its + SPDK does not answer.""" + try: + return bool(rpc.bdev_get(dev.alceml_bdev)) or bool(rpc.bdev_get(f"{dev.alceml_bdev}_PT")) + except Exception: # noqa: BLE001 + return None + + # ------------------------------------------------------------- operator def unfence_node(device_id, node_id) -> bool: diff --git a/simplyblock_core/services/main_distr_event_collector.py b/simplyblock_core/services/main_distr_event_collector.py index 5a92713601..d0c7d00030 100644 --- a/simplyblock_core/services/main_distr_event_collector.py +++ b/simplyblock_core/services/main_distr_event_collector.py @@ -10,6 +10,7 @@ constants, db_controller, distr_controller, + multi_attach, rpc_client, utils, ) @@ -399,7 +400,12 @@ def process_device_event(event, logger): return - if device_node_obj.get_id() == event_node_obj.get_id(): + # On multi-attach the home node of a LUN served by another node reads + # it remotely like any peer: its error is about that path, not the + # LUN (2026-11-20 FTT2 bed: a home node's error_read on the dead + # serving node's export, processed after the takeover, marked the + # taken-over LUN unavailable cluster-wide and blocked every LVS). + if device_node_obj.get_id() == event_node_obj.get_id() and not multi_attach.served_elsewhere(device_obj, event_node_obj.get_id()): # The device's home node itself reported an IO error against its # local device — this is the only path that may count toward the # per-device flap budget. The CAUSE_LOCAL_FAILURE opt-in is diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index f3fa7f9518..b99cd90d1f 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -3209,6 +3209,7 @@ def _retire_taken_over_luns(snode): _reintegrate_last: dict = {} +_reintegrate_threads: dict = {} REINTEGRATE_INTERVAL_SEC = 30 @@ -3224,12 +3225,22 @@ def _reintegrate_fenced_luns(snode): now = time.monotonic() if now - _reintegrate_last.get(node_id, 0) < REINTEGRATE_INTERVAL_SEC: return + t = _reintegrate_threads.get(node_id) + if t is not None and t.is_alive(): + return _reintegrate_last[node_id] = now - try: - from simplyblock_core.controllers import lun_takeover - lun_takeover.reintegrate_fenced_node(snode) - except Exception as e: # noqa: BLE001 - logger.error("Reintegrating fenced LUNs of %s failed: %s", node_id, e) + + def _run(): + try: + from simplyblock_core.controllers import lun_takeover + lun_takeover.reintegrate_fenced_node(snode) + except Exception as e: # noqa: BLE001 + logger.error("Reintegrating fenced LUNs of %s failed: %s", node_id, e) + # Off the node check: its agent calls on a node that is just lost held + # the liveness checks 30 s (2026-11-20 FTT2 bed, hard reset of sn3). + t = threading.Thread(target=_run, name=f"reintegrate-{node_id[:8]}", daemon=True) + _reintegrate_threads[node_id] = t + t.start() def readmit_devices_after_node_online(node_id): diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index bee89881b7..1e07c6c99f 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -13,6 +13,8 @@ """ import copy +import threading +import time import unittest from unittest.mock import MagicMock, patch @@ -891,6 +893,65 @@ def test_stale_stack_is_retired_first(self): self.assertEqual(self.dev(db).fenced_node_ids, []) + def test_stays_fenced_while_its_array_fence_runs(self): + # 2026-11-20 resize bed, R3: unfenced while the background detach + # ran; the detach then left the node off the LUN for good. + db, a = self._taken_over() + key = (DEV_ID, "node-a") + with lun_takeover._fences_guard: + lun_takeover._fences_in_flight.add(key) + try: + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + finally: + with lun_takeover._fences_guard: + lun_takeover._fences_in_flight.discard(key) + self.assertEqual(self.dev(db).fenced_node_ids, ["node-a"]) + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), [DEV_ID]) + + def test_stays_fenced_when_spdk_does_not_answer(self): + # A node just lost can still read ONLINE; its dead SPDK proves + # nothing about a stale stack. + db, a = self._taken_over() + a.spdk.bdev_get = MagicMock(side_effect=ConnectionError("refused")) + self.assertEqual(lun_takeover.reintegrate_fenced_node(a), []) + self.assertEqual(self.dev(db).fenced_node_ids, ["node-a"]) + a.agent.lun_info.assert_not_called() + + +class TestFenceAttachmentRecord(_TakeoverCase): + """An array fence drops the node from array_attached_node_ids, so the + attachment reconcile presents the LUN again once it is unfenced.""" + + def test_array_fence_forgets_the_attachment(self): + db, dev = _bed() + dev.array_attached_node_ids = ["node-a", "node-b", "node-c"] + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).array_attached_node_ids, ["node-b", "node-c"]) + + def test_background_fence_is_in_flight_until_done(self): + db, dev = _bed() + dev.lun_reservations = True + dev.array_attached_node_ids = ["node-a", "node-b"] + gate = threading.Event() + fence = FakeArrayPlugin.fence + + def slow(plugin, *a): + gate.wait(5) + return fence(plugin, *a) + b = db.nodes["node-b"] + b.spdk.alceml_get_role = lambda name: {"role": "active", "state": "active"} + with patch.object(FakeArrayPlugin, "fence", slow): + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertTrue(lun_takeover.fence_in_flight(DEV_ID, "node-a")) + gate.set() + for _ in range(250): + if not lun_takeover.fence_in_flight(DEV_ID, "node-a"): + break + time.sleep(0.02) + self.assertFalse(lun_takeover.fence_in_flight(DEV_ID, "node-a")) + self.assertEqual(self.dev(db).array_attached_node_ids, ["node-b"]) + + class TestUnfenceOrder(_TakeoverCase): """The fence record is cleared before the array call: the attachment reconcile detaches any host the record still lists as fenced.""" diff --git a/tests/unit/test_multi_attach_device_events.py b/tests/unit/test_multi_attach_device_events.py new file mode 100644 index 0000000000..a897e4fde6 --- /dev/null +++ b/tests/unit/test_multi_attach_device_events.py @@ -0,0 +1,70 @@ +"""Distrib device events on multi-attach clusters: the home node of a LUN +served by another node reaches it remotely, like any peer. Its I/O error is +about that path and must not fail the LUN for the whole cluster (2026-11-20 +FTT2 bed: a home node's error_read on the dead serving node's export, +processed after the takeover, marked the taken-over LUN unavailable and +blocked every LVS of the 1+0 cluster).""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import main_distr_event_collector as collector + + +def _run(serving_node_id, multi=True): + dev = NVMeDevice() + dev.uuid = "dev-1" + dev.node_id = "node-1" + dev.cluster_device_order = 3 + dev.status = NVMeDevice.STATUS_ONLINE + dev.multi_attach = multi + dev.serving_node_id = serving_node_id + + home = MagicMock() + home.get_id.return_value = "node-1" + home.status = StorageNode.STATUS_ONLINE + home.cluster_id = "c1" + home.nvme_devices = [dev] + + event = MagicMock() + event.message = "error_read" + event.node_id = "node-1" + event.storage_id = 3 + event.object_dict = {} + + db = MagicMock() + db.get_storage_node_by_id.return_value = home + db.get_storage_nodes_by_cluster_id.return_value = [home] + with patch.object(collector, "db", db), \ + patch.object(collector.device_controller, "device_set_unavailable") as unavailable, \ + patch.object(collector.device_controller, "device_set_io_error"), \ + patch.object(collector.distr_controller, "send_dev_status_event") as peer_event, \ + patch.object(collector, "remove_remote_device_from_node"), \ + patch.object(NVMeDevice, "lock_device_connection"), \ + patch.object(NVMeDevice, "release_device_connection"), \ + patch.object(NVMeDevice, "is_connection_in_progress_to_node", return_value=False): + collector.process_device_event(event, collector.logger) + return unavailable, peer_event + + +class TestHomeNodeOfLunServedElsewhere(unittest.TestCase): + + def test_error_is_handled_as_a_peer_path_error(self): + unavailable, peer_event = _run(serving_node_id="node-2") + unavailable.assert_not_called() + peer_event.assert_called_once() + + def test_home_node_serving_its_lun_fails_it(self): + unavailable, peer_event = _run(serving_node_id="") + unavailable.assert_called_once() + peer_event.assert_not_called() + + def test_regular_cluster_unchanged(self): + unavailable, _ = _run(serving_node_id="node-2", multi=False) + unavailable.assert_called_once() + + +if __name__ == "__main__": + unittest.main() From 17a8b9103d3db6fa79dad05680b5557549b48ea3 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 18:10:43 +0100 Subject: [PATCH 34/50] fix(array): netapp-ontap derives protocol_family from the protocol An NVMe protocol without protocol_family created SCSI LUNs that the NVMe hosts never see. The family now follows the plugin's protocol, and a stated family that contradicts it is refused. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/array/san.py | 9 +++++++++ tests/unit/storage_arrays/test_san_plugin.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/simplyblock_core/array/san.py b/simplyblock_core/array/san.py index 4950687f4f..ed5a68f700 100644 --- a/simplyblock_core/array/san.py +++ b/simplyblock_core/array/san.py @@ -152,6 +152,15 @@ def connect(self, endpoint: str, credentials: dict, options: dict) -> None: raise ArrayError("the plugin has no cluster id") proto = self.protocol adapter_opts = {k: v for k, v in self.options.items() if k not in _PLUGIN_OPTIONS} + if self.vendor == "netapp-ontap": + # ONTAP creates SCSI LUNs or NVMe namespaces; which one follows + # from the protocol the hosts use, so it is derived, not repeated. + family = "nvme" if proto.is_nvme else "scsi" + stated = str(adapter_opts.get("protocol_family") or family).lower() + if stated != family: + raise ArrayError(f"netapp-ontap: protocol_family {stated!r} contradicts protocol " + f"{proto.value!r} (which needs {family!r})") + adapter_opts["protocol_family"] = family try: config = sa.ArrayConfig( id=f"{self.cluster_id}", diff --git a/tests/unit/storage_arrays/test_san_plugin.py b/tests/unit/storage_arrays/test_san_plugin.py index e88bdfe14c..87e7836b38 100644 --- a/tests/unit/storage_arrays/test_san_plugin.py +++ b/tests/unit/storage_arrays/test_san_plugin.py @@ -152,3 +152,17 @@ def test_names(): assert len(long) == 31 and long.startswith("sb-x") assert san.host_object_name("x" * 80, 31) == long assert san.host_object_name("y" * 80, 31) != long + + +# NetApp creates SCSI LUNs unless told otherwise; an NVMe protocol without +# protocol_family would create LUNs the NVMe hosts never see. +def test_netapp_protocol_family_follows_the_protocol(): + bed = Bed("netapp-ontap") + bed.protocol = Protocol.NVME_TCP + assert bed.plugin().client.config.options["protocol_family"] == "nvme" + bed.protocol = Protocol.ISCSI + assert bed.plugin().client.config.options.get("protocol_family", "scsi") == "scsi" + # A stated family that contradicts the protocol is refused. + bed.protocol = Protocol.NVME_TCP + with pytest.raises(array.ArrayError, match="protocol_family"): + bed.plugin(protocol_family="scsi") From f9b9795289608d8f4cdbb1ba4ccf477185974753 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 18:20:36 +0100 Subject: [PATCH 35/50] fix(storage_arrays): read string options as the type of their default Options set through Kubernetes (StorageCluster.spec.multiAttach.array.options) arrive as strings. Adapters tested booleans by truthiness, so eradicate: "false" on Pure eradicated deleted volumes, thin: "false" on IBM stayed thin, and Hitachi host_mode_options "2,13" failed. ArrayAdapter.option now reads a string as its default's type (true/false/yes/no/on/off/1/0, numbers, comma-separated lists) and a leading { as JSON (PowerMax port_group map); a value that cannot be read fails at registration. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/storage_arrays/base.py | 51 +++++++++++++++++++-- tests/unit/storage_arrays/test_framework.py | 43 +++++++++++++++++ 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/simplyblock_core/storage_arrays/base.py b/simplyblock_core/storage_arrays/base.py index 8d2a6b927e..c289df84cc 100644 --- a/simplyblock_core/storage_arrays/base.py +++ b/simplyblock_core/storage_arrays/base.py @@ -10,6 +10,7 @@ """ from __future__ import annotations +import json import re from abc import ABC, abstractmethod from collections.abc import Mapping as MappingABC @@ -234,6 +235,40 @@ class OptionDoc: default: Any = None +_TRUE = frozenset({"true", "yes", "on", "1"}) +_FALSE = frozenset({"false", "no", "off", "0"}) + + +def _coerce_option(owner: str, name: str, value: Any, default: Any) -> Any: + if not isinstance(value, str): + return value + text = value.strip() + if text.startswith("{"): + try: + return json.loads(text) + except ValueError: + raise ArrayError(f"{owner} option {name!r} is not valid JSON") from None + if isinstance(default, bool): + if text.lower() in _TRUE: + return True + if text.lower() in _FALSE: + return False + raise ArrayError(f"{owner} option {name!r} must be true or false, got {value!r}") + if isinstance(default, int): + try: + return int(text) + except ValueError: + raise ArrayError(f"{owner} option {name!r} must be a whole number, got {value!r}") from None + if isinstance(default, float): + try: + return float(text) + except ValueError: + raise ArrayError(f"{owner} option {name!r} must be a number, got {value!r}") from None + if isinstance(default, (list, tuple)): + return [item for item in (part.strip() for part in text.replace(",", " ").split()) if item] + return value + + @dataclass class AdapterContext: """What an adapter is constructed with.""" @@ -299,13 +334,21 @@ def _check_options(self) -> None: unknown = sorted(set(self.options) - known) if unknown: raise ArrayError(f"Unknown {self.display_name} options: {', '.join(unknown)}") + for o in self.option_docs: + if o.name in self.options: + self.option(o.name) # a value that cannot be read fails here, at registration def option(self, name: str) -> Any: + """The option's value, or its default. A string (options set through + Kubernetes are strings) is read as the type of the default: a boolean + from true/false/yes/no/on/off/1/0, a number, a comma-separated list, + and a JSON object where it starts with ``{``.""" + doc = next((o for o in self.option_docs if o.name == name), None) if name in self.options: - return self.options[name] - for o in self.option_docs: - if o.name == name: - return o.default + return _coerce_option(self.display_name, name, self.options[name], + doc.default if doc else None) + if doc is not None: + return doc.default raise KeyError(name) def secret(self, name: str) -> str: diff --git a/tests/unit/storage_arrays/test_framework.py b/tests/unit/storage_arrays/test_framework.py index 5468600432..ef3b4e7360 100644 --- a/tests/unit/storage_arrays/test_framework.py +++ b/tests/unit/storage_arrays/test_framework.py @@ -221,3 +221,46 @@ def test_end_to_end_over_tls_with_pinned_ca(self): def test_requests_session_is_plain(self): # FakeTransport is mounted per session, never globally. assert not isinstance(requests.Session().get_adapter("https://x"), FakeTransport) + + +# Options set through Kubernetes arrive as strings. Each is read as the type of +# its default, so "false" is false rather than a truthy string. +def test_string_options_are_read_as_the_type_of_their_default(): + from unittest.mock import MagicMock + + from pydantic import SecretStr + + from simplyblock_core.credentials import Credential + from simplyblock_core.storage_arrays.adapters.dell_powermax import ( + DellPowerMaxAdapter, + ) + from simplyblock_core.storage_arrays.adapters.hitachi_vsp import HitachiVspAdapter + from simplyblock_core.storage_arrays.adapters.pure import PureFlashArrayAdapter + from simplyblock_core.storage_arrays.base import AdapterContext, ArrayError + from simplyblock_core.storage_arrays.testing import make_config + + def adapter(cls, vendor, creds, **options): + return cls(AdapterContext(make_config(vendor, "https://a.example", **options), + Credential({k: SecretStr(v) for k, v in creds.items()}), MagicMock())) + + pure = adapter(PureFlashArrayAdapter, "pure-flasharray", {"api_token": "t"}, eradicate="false") + assert pure.option("eradicate") is False + assert adapter(PureFlashArrayAdapter, "pure-flasharray", {"api_token": "t"}, + eradicate="True").option("eradicate") is True + with pytest.raises(ArrayError, match="eradicate"): + adapter(PureFlashArrayAdapter, "pure-flasharray", {"api_token": "t"}, eradicate="maybe") + + up = {"username": "u", "password": "p"} + vsp = adapter(HitachiVspAdapter, "hitachi-vsp", up, pool_id="1", ports="CL1-A", + host_mode_options="2, 13", job_timeout_s="60", use_lock="0") + assert vsp.option("host_mode_options") == ["2", "13"] + assert vsp.option("job_timeout_s") == 60 + assert vsp.option("use_lock") is False + + pmax = adapter(DellPowerMaxAdapter, "dell-powermax", up, symmetrix_id="000197900123", + port_group='{"iscsi": "pg_iscsi", "fc": "pg_fc"}', async_jobs="no") + assert pmax.option("port_group") == {"iscsi": "pg_iscsi", "fc": "pg_fc"} + assert pmax.option("async_jobs") is False + plain = adapter(DellPowerMaxAdapter, "dell-powermax", up, symmetrix_id="000197900123", + port_group="pg_all") + assert plain.option("port_group") == "pg_all" From 183aa3d3f30ba056dcf89c4ea00c6a6a11f8588e Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 19:17:19 +0100 Subject: [PATCH 36/50] test(storage_arrays): mock arrays with a real iSCSI data path (LIO) The vendor fakes answer the management API but present nothing to a host. testing/lio.py mirrors a fake's volumes, hosts and mappings into the kernel's iSCSI target: every volume a loop-backed block LUN, every host initiator an ACL, every mapping an ACL LUN, an unmap a real fence. Sizes and identities are read through the vendor's own adapter in process, so the host sees what the array plug-in reports (identity as the LUN's unit serial, lsblk SERIAL). - fakes: lio_volume_ids / lio_host_iqns / lio_all_iqns (10 iSCSI vendors). - serve(after_request=...): reconcile after every request; jobs of asynchronous arrays (Hitachi) complete on GET polls. - mock server: --lio, --lio-target, --lio-portal, --lio-dir, --lio-allow. - testing/lio_selftest.py: the multi-attach plug-in (array.san) against the mock over HTTPS with two iSCSI initiators of one host as two nodes, the node agent's scan_for_new_luns / find_lun_devices / rescan_block_device on real block devices: attach, multi-attach data, fence (I/O fails), growth, unfence, delete. Passes 9/9 for all 10 iSCSI vendors (WSL2, kernel LIO). - test_lio.py: the mirror against MemoryLio for every vendor. Co-Authored-By: Claude Opus 5.5 --- .../storage_arrays/testing/__init__.py | 26 +- .../storage_arrays/testing/__main__.py | 37 +- .../storage_arrays/testing/dell_powerflex.py | 11 + .../storage_arrays/testing/dell_powermax.py | 12 + .../storage_arrays/testing/dell_powerstore.py | 14 + .../storage_arrays/testing/hitachi_vsp.py | 15 + .../storage_arrays/testing/hpe_nimble.py | 13 + .../storage_arrays/testing/hpe_wsapi.py | 12 + .../storage_arrays/testing/huawei_dorado.py | 15 + .../storage_arrays/testing/ibm_flashsystem.py | 14 + .../storage_arrays/testing/infinidat.py | 14 + .../storage_arrays/testing/lio.py | 331 ++++++++++++++++++ .../storage_arrays/testing/lio_selftest.py | 265 ++++++++++++++ .../storage_arrays/testing/netapp_ontap.py | 14 + .../storage_arrays/testing/pure.py | 12 + tests/unit/storage_arrays/test_lio.py | 105 ++++++ 16 files changed, 904 insertions(+), 6 deletions(-) create mode 100644 simplyblock_core/storage_arrays/testing/lio.py create mode 100644 simplyblock_core/storage_arrays/testing/lio_selftest.py create mode 100644 tests/unit/storage_arrays/test_lio.py diff --git a/simplyblock_core/storage_arrays/testing/__init__.py b/simplyblock_core/storage_arrays/testing/__init__.py index 67f49ad7e2..cf82bb245e 100644 --- a/simplyblock_core/storage_arrays/testing/__init__.py +++ b/simplyblock_core/storage_arrays/testing/__init__.py @@ -23,6 +23,7 @@ import http.client import io import json +import logging import re import threading import types @@ -148,6 +149,17 @@ def not_routed(self, req: Request) -> Reply: return Reply(404, {"error": f"fake: no route for {req.method} {req.path}"}) +# The LIO data path (testing.lio) reads a fake through two accessors, which +# every vendor fake implements: +# +# * ``lio_volume_ids()``: the ids of the live volumes, as the vendor's adapter +# reports them in ``Volume.id``; +# * ``lio_host_iqns(host_id)``: the iSCSI initiator names of the host the +# adapter knows as ``host_id`` (the id ``ensure_host`` returned or the name +# ``list_mappings`` reports), lower case; +# * ``lio_all_iqns()``: the iSCSI initiator names of every host, so that a +# registered host can log in before anything is mapped to it. + class FakeError(Exception): """Raise inside a handler to answer with ``reply``.""" @@ -254,10 +266,15 @@ def make_config(vendor: str, endpoint: str, **options: Any) -> ArrayConfig: def serve(fake: FakeArray, host: str = "127.0.0.1", port: int = 8443, - certfile: str | None = None, keyfile: str | None = None): + certfile: str | None = None, keyfile: str | None = None, + after_request: Callable[[], None] | None = None): """Serve ``fake`` over HTTPS (a throwaway self-signed certificate is generated unless ``certfile``/``keyfile`` are given). Returns the server; - call ``serve_forever()`` on it.""" + call ``serve_forever()`` on it. ``after_request`` runs after every + request, before the answer is sent: the LIO data path (``testing.lio``) + catches up there, so a mapping the caller was told about is already + presented. GETs included: on arrays with asynchronous jobs, a change + takes effect while its job is polled.""" import ssl import tempfile from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -273,6 +290,11 @@ def _do(self): method=self.command, path=parts.path, query=parse_qs(parts.query, keep_blank_values=True), headers=CaseInsensitiveDict(dict(self.headers.items())), body=body)) + if after_request is not None: + try: + after_request() + except Exception: # noqa: BLE001 - the API answer still goes out + logging.getLogger(__name__).exception("after_request failed") content, headers = _encode(reply) self.send_response(reply.status) for k, v in headers.items(): diff --git a/simplyblock_core/storage_arrays/testing/__main__.py b/simplyblock_core/storage_arrays/testing/__main__.py index 6eee1688aa..95ce4009ce 100644 --- a/simplyblock_core/storage_arrays/testing/__main__.py +++ b/simplyblock_core/storage_arrays/testing/__main__.py @@ -4,6 +4,14 @@ Prints the accepted credential, the CA certificate path (pass its content as the array's ``ca_bundle``) and the options to register the array with. + +With ``--lio`` (root, LIO modules and python3-rtslib-fb on the host) the +fake's volumes are real iSCSI LUNs of the kernel target, presented to the +hosts the fake maps them to (see ``testing.lio``):: + + python -m simplyblock_core.storage_arrays.testing --vendor pure-flasharray \ + --host 0.0.0.0 --lio --lio-portal 10.0.0.5:3260 \ + --lio-allow iqn.1994-05.com.redhat:node1,iqn.1994-05.com.redhat:node2 """ import argparse import importlib @@ -17,13 +25,32 @@ def main() -> None: parser.add_argument("--vendor", required=True, help=", ".join(sorted(ADAPTERS))) parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8443) + parser.add_argument("--lio", action="store_true", + help="present the volumes as iSCSI LUNs of the kernel target") + parser.add_argument("--lio-target", default="iqn.2026-11.io.simplyblock:mock-array", + help="IQN of the iSCSI target") + parser.add_argument("--lio-portal", default="0.0.0.0:3260", help="portal address:port") + parser.add_argument("--lio-dir", default="/var/lib/sb-mock-lio", help="where the LUN files live") + parser.add_argument("--lio-allow", default="", + help="comma-separated initiators allowed to log in before they are registered") args = parser.parse_args() vendor = canonical_vendor(args.vendor) module = importlib.import_module( f"{__package__}.{ADAPTERS[vendor].split(':')[0].split('.')[-1]}") setup = module.contract_setup() - server = serve_fake(setup.fake, args.host, args.port) + mirror = None + if args.lio: + from . import FakeTransport + from .lio import ConfigfsLio, LioMirror + ip, _, port = args.lio_portal.rpartition(":") + backend = ConfigfsLio(args.lio_target, args.lio_dir, (ip or "0.0.0.0", int(port))) + mirror = LioMirror(setup.fake, setup.config, setup.credential, backend, + transport=FakeTransport(setup.fake), + allow=[i for i in args.lio_allow.split(",") if i]) + mirror.reconcile() + server = serve_fake(setup.fake, args.host, args.port, + mirror.reconcile if mirror is not None else None) print(json.dumps({ "vendor": vendor, "endpoint": f"https://{args.host}:{args.port}", @@ -32,16 +59,18 @@ def main() -> None: "credential": {k: setup.credential.get(k).get_secret_value() for k in setup.credential.field_names}, "options": setup.config.options, - }, indent=2)) + "iscsi_target": args.lio_target if args.lio else None, + "iscsi_portal": args.lio_portal if args.lio else None, + }, indent=2), flush=True) try: server.serve_forever() except KeyboardInterrupt: pass -def serve_fake(fake, host, port): +def serve_fake(fake, host, port, after_request=None): from . import serve - return serve(fake, host, port) + return serve(fake, host, port, after_request=after_request) if __name__ == "__main__": diff --git a/simplyblock_core/storage_arrays/testing/dell_powerflex.py b/simplyblock_core/storage_arrays/testing/dell_powerflex.py index 4e08d97947..b6897db25e 100644 --- a/simplyblock_core/storage_arrays/testing/dell_powerflex.py +++ b/simplyblock_core/storage_arrays/testing/dell_powerflex.py @@ -108,6 +108,17 @@ def _auth(self, req: Request, basic_ok: bool = False) -> None: return raise _err("Unauthorized", 401) + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return [] # PowerFlex presents through its SDC, not iSCSI + + def lio_all_iqns(self) -> list[str]: + return sorted(set()) + + def lio_host_iqns(self, host_id: str) -> list[str]: + return [] + @route("GET", "/api/version") def api_version(self, req): self._auth(req, basic_ok=True) diff --git a/simplyblock_core/storage_arrays/testing/dell_powermax.py b/simplyblock_core/storage_arrays/testing/dell_powermax.py index 177139b58c..2c4903481b 100644 --- a/simplyblock_core/storage_arrays/testing/dell_powermax.py +++ b/simplyblock_core/storage_arrays/testing/dell_powermax.py @@ -146,6 +146,18 @@ def _run(self, req: Request, fn: Callable[[], Reply]) -> Reply: def _public_job(self, job_id: str) -> dict[str, Any]: return {k: v for k, v in self.jobs[job_id].items() if not k.startswith("_")} + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.volumes) + + def lio_all_iqns(self) -> list[str]: + return sorted({str(i).lower() for h in self.hosts.values() for i in h.get("initiator", []) if str(i).lower().startswith("iqn.")}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + host = self.hosts.get(host_id) or {} + return [str(i).lower() for i in host.get("initiator", []) if str(i).lower().startswith("iqn.")] + @route("GET", _V + r"/system/job/(?P\d+)") def get_job(self, req, ver, job_id): self._check(req, ver) diff --git a/simplyblock_core/storage_arrays/testing/dell_powerstore.py b/simplyblock_core/storage_arrays/testing/dell_powerstore.py index 631871d018..4993abe1b2 100644 --- a/simplyblock_core/storage_arrays/testing/dell_powerstore.py +++ b/simplyblock_core/storage_arrays/testing/dell_powerstore.py @@ -123,6 +123,20 @@ def _session(self, req: Request) -> None: if req.method != "GET" and req.headers.get("DELL-EMC-TOKEN") != self.sessions[cookie]: raise _err(403, "Forbidden: missing or invalid DELL-EMC-TOKEN.", "0xE09040010003") + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.volumes) + + def lio_all_iqns(self) -> list[str]: + return sorted({p["port_name"].lower() for h in self.hosts.values() for p in h["host_initiators"] if p["port_type"] == "iSCSI"}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + for h in self.hosts.values(): + if host_id in (h["id"], h["name"]): + return [p["port_name"].lower() for p in h["host_initiators"] if p["port_type"] == "iSCSI"] + return [] + @route("GET", API + "/login_session") def login_session(self, req): if req.basic_auth() != (self.username, self.password): diff --git a/simplyblock_core/storage_arrays/testing/hitachi_vsp.py b/simplyblock_core/storage_arrays/testing/hitachi_vsp.py index 77f63c11cf..ce78042cb4 100644 --- a/simplyblock_core/storage_arrays/testing/hitachi_vsp.py +++ b/simplyblock_core/storage_arrays/testing/hitachi_vsp.py @@ -144,6 +144,21 @@ def _auth(self, req: Request, sdid: str | None) -> str: self._prefix(sdid) return self._token(req) + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return [str(i) for i in self.ldevs] + + def lio_all_iqns(self) -> list[str]: + return sorted({str(i).lower() for g in self.groups.values() for i in g.get("iscsis", [])}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + out: list[str] = [] + for g in self.groups.values(): + if g["name"] == host_id: + out += [str(i).lower() for i in g.get("iscsis", []) if str(i).lower() not in out] + return out + @route("POST", _P + "/sessions") def post_session(self, req, sdid): self._prefix(sdid) diff --git a/simplyblock_core/storage_arrays/testing/hpe_nimble.py b/simplyblock_core/storage_arrays/testing/hpe_nimble.py index 350d841cc9..e03cbd5982 100644 --- a/simplyblock_core/storage_arrays/testing/hpe_nimble.py +++ b/simplyblock_core/storage_arrays/testing/hpe_nimble.py @@ -144,6 +144,19 @@ def _done(self, op: str, status: int, data: Any) -> Reply: "text": f"Job {jid} is running", "arguments": {"job_id": jid}}]}) + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.volumes) + + def lio_all_iqns(self) -> list[str]: + return sorted({i["iqn"].lower() for i in self.initiators.values() if i.get("iqn")}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + gids = {g["id"] for g in self.groups.values() if host_id in (g["id"], g["name"])} + return [i["iqn"].lower() for i in self.initiators.values() + if i["initiator_group_id"] in gids and i.get("iqn")] + @route("GET", "/v1/jobs/" + _ID) def get_job(self, req, obj_id): self._auth(req) diff --git a/simplyblock_core/storage_arrays/testing/hpe_wsapi.py b/simplyblock_core/storage_arrays/testing/hpe_wsapi.py index 9a3ffcb911..36d8c559ff 100644 --- a/simplyblock_core/storage_arrays/testing/hpe_wsapi.py +++ b/simplyblock_core/storage_arrays/testing/hpe_wsapi.py @@ -90,6 +90,18 @@ def _auth(self, req: Request) -> None: if req.headers.get("X-HP3PAR-WSAPI-SessionKey") not in self.sessions: raise _err(401, 6, "invalid session key") + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.volumes) + + def lio_all_iqns(self) -> list[str]: + return sorted({p["name"].lower() for h in self.hosts.values() for p in h.get("iSCSIPaths", [])}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + host = self.hosts.get(host_id) or {} + return [p["name"].lower() for p in host.get("iSCSIPaths", [])] + @route("POST", _A + "/credentials") def login(self, req): body = req.json() or {} diff --git a/simplyblock_core/storage_arrays/testing/huawei_dorado.py b/simplyblock_core/storage_arrays/testing/huawei_dorado.py index 8134183a5c..0917185695 100644 --- a/simplyblock_core/storage_arrays/testing/huawei_dorado.py +++ b/simplyblock_core/storage_arrays/testing/huawei_dorado.py @@ -131,6 +131,21 @@ def _auth(self, req: Request, dev: str) -> None: if dev != self.device_id or req.headers.get("iBaseToken") not in self.tokens: raise _fail(UNAUTHORIZED, "The user is not logged in or the session has expired.") + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.luns) + + def lio_all_iqns(self) -> list[str]: + return sorted({i["ID"].lower() for i in self.initiators.get("iscsi_initiator", {}).values() if i["PARENTID"]}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + hid = next((h["ID"] for h in self.hosts.values() if host_id in (h["ID"], h["NAME"])), None) + if hid is None: + return [] + return [i["ID"].lower() for i in self.initiators.get("iscsi_initiator", {}).values() + if i["PARENTID"] == hid] + @route("POST", r"/deviceManager/rest/xx/sessions") def login(self, req): body = req.json() or {} diff --git a/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py b/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py index 005999ef1c..3446b2f278 100644 --- a/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py +++ b/simplyblock_core/storage_arrays/testing/ibm_flashsystem.py @@ -119,6 +119,20 @@ def add_volume(self, name: str, size: int, thin: bool = False) -> str: # -- dispatch ------------------------------------------------------------------- + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.vdisks) + + def lio_all_iqns(self) -> list[str]: + return sorted({str(i).lower() for h in self.hosts.values() for i in h.get("iscsiname", [])}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + for name, h in self.hosts.items(): + if host_id in (name, str(h.get("id", ""))): + return [str(i).lower() for i in h.get("iscsiname", [])] + return [] + @route("POST", _ROOT + "/auth") def auth(self, req: Request): if (req.headers.get("X-Auth-Username") != self.username diff --git a/simplyblock_core/storage_arrays/testing/infinidat.py b/simplyblock_core/storage_arrays/testing/infinidat.py index 2412cfa398..d319c90ec5 100644 --- a/simplyblock_core/storage_arrays/testing/infinidat.py +++ b/simplyblock_core/storage_arrays/testing/infinidat.py @@ -147,6 +147,20 @@ def _lun_view(self, m: dict[str, Any]) -> dict[str, Any]: # -- system / pools -------------------------------------------------------------- + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return [str(i) for i in self.volumes] + + def lio_all_iqns(self) -> list[str]: + return sorted({p["address"].lower() for h in self.hosts.values() for p in h["ports"] if p["type"] == "ISCSI"}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + for h in self.hosts.values(): + if host_id in (str(h["id"]), h["name"]): + return [p["address"].lower() for p in h["ports"] if p["type"] == "ISCSI"] + return [] + @route("GET", _R + "/system") def system(self, req): self._auth(req) diff --git a/simplyblock_core/storage_arrays/testing/lio.py b/simplyblock_core/storage_arrays/testing/lio.py new file mode 100644 index 0000000000..8ba740b77c --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/lio.py @@ -0,0 +1,331 @@ +"""The data path of a mock array: a vendor fake's volumes, hosts, and +mappings mirrored into the Linux kernel's iSCSI target (LIO). + +The fakes answer the vendor's management API and keep the array's state in +memory, which is enough to test the adapters but presents nothing to a host. +With a mirror attached, every volume of the fake becomes a real block device +exported over iSCSI, every host's initiator an ACL of the target, and every +mapping a LUN of that ACL. A host logged in to the target therefore sees a LUN +appear after a mapping and a SCSI rescan, lose it when it is unmapped (the +fence of a takeover), and see it grow after a resize, exactly as with an +array. This is what lets the control plane's whole multi-attach path (node +add, standby presentation, fencing, takeover, growth) run against a mock. + +The mirror reads the fake through the vendor's own adapter, in process, so a +LUN's size and identity are what the array plug-in reports for it. The +identity reaches the host as the LUN's SCSI unit serial (lsblk SERIAL), which +LUN matching accepts beside the WWN. Only the volume ids and the hosts' +initiator names are read from the fake directly (``lio_volume_ids``, +``lio_host_iqns``, ``lio_all_iqns``). + +Backends: :class:`MemoryLio` for tests, :class:`ConfigfsLio` for the kernel +target (root, ``target_core_iblock``, ``iscsi_target_mod``, and +``python3-rtslib-fb``). Volumes are sparse files on loop devices, exported as +block backstores, so a resize is a truncate and a loop capacity refresh. +""" +from __future__ import annotations + +import hashlib +import logging +import os +import subprocess +import threading +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Any +from typing import Protocol as TypingProtocol + +from simplyblock_core.credentials import Credential + +from ..base import ArrayConfig, Protocol +from ..client import ArrayClient + +logger = logging.getLogger(__name__) + +KEY_PREFIX = "sbm_" + + +@dataclass +class LioVolume: + size_bytes: int + serial: str + + +class LioBackend(TypingProtocol): + def list_volumes(self) -> dict[str, LioVolume]: ... + def create_volume(self, key: str, size_bytes: int, serial: str) -> None: ... + def resize_volume(self, key: str, size_bytes: int) -> None: ... + def delete_volume(self, key: str) -> None: ... + def list_acls(self) -> dict[str, dict[int, str]]: ... + def ensure_acl(self, iqn: str) -> None: ... + def delete_acl(self, iqn: str) -> None: ... + def map_lun(self, iqn: str, mapped_lun: int, key: str) -> None: ... + def unmap_lun(self, iqn: str, mapped_lun: int) -> None: ... + def close(self) -> None: ... + + +@dataclass +class MemoryLio: + """The target as dictionaries: volumes by key, and per ACL its LUNs.""" + + volumes: dict[str, LioVolume] = field(default_factory=dict) + acls: dict[str, dict[int, str]] = field(default_factory=dict) + #: Count of changes, for idempotence checks. + ops: int = 0 + + def list_volumes(self) -> dict[str, LioVolume]: + return {k: LioVolume(v.size_bytes, v.serial) for k, v in self.volumes.items()} + + def create_volume(self, key: str, size_bytes: int, serial: str) -> None: + self.ops += 1 + self.volumes[key] = LioVolume(size_bytes, serial) + + def resize_volume(self, key: str, size_bytes: int) -> None: + self.ops += 1 + self.volumes[key].size_bytes = size_bytes + + def delete_volume(self, key: str) -> None: + self.ops += 1 + assert all(key not in luns.values() for luns in self.acls.values()), "volume still mapped" + del self.volumes[key] + + def list_acls(self) -> dict[str, dict[int, str]]: + return {iqn: dict(luns) for iqn, luns in self.acls.items()} + + def ensure_acl(self, iqn: str) -> None: + if iqn not in self.acls: + self.ops += 1 + self.acls[iqn] = {} + + def delete_acl(self, iqn: str) -> None: + self.ops += 1 + del self.acls[iqn] + + def map_lun(self, iqn: str, mapped_lun: int, key: str) -> None: + self.ops += 1 + assert key in self.volumes + self.acls[iqn][mapped_lun] = key + + def unmap_lun(self, iqn: str, mapped_lun: int) -> None: + self.ops += 1 + del self.acls[iqn][mapped_lun] + + def close(self) -> None: + pass + + +class _StaticCredentials: + def __init__(self, credential: Credential): + self._credential = credential + + def resolve(self, ref, context): + return self._credential + + def invalidate(self, ref=None): + pass + + +def _host_identity(vol) -> str: + """What the array plug-in reports for a SCSI LUN (array.san.host_wwn).""" + from simplyblock_core.array.san import host_wwn + return host_wwn(vol, Protocol.ISCSI) + + +class LioMirror: + """Keeps a backend in step with a vendor fake. ``reconcile`` is cheap and + idempotent, and is called after every request the mock server answers.""" + + def __init__(self, fake: Any, config: ArrayConfig, credential: Credential, backend: LioBackend, + *, transport: Any, session_kwargs: dict[str, Any] | None = None, + allow: Iterable[str] = ()): + self.fake = fake + self.backend = backend + #: Initiators given an ACL whether or not a host of the array has them: + #: the hosts log in to the target before they are registered, as they + #: would to an array that presents nothing to an unknown initiator. + self.allow = [i.lower() for i in allow] + self.client = ArrayClient(config, _StaticCredentials(credential), # type: ignore[arg-type] + transport=transport, session_kwargs=session_kwargs) + self._lock = threading.Lock() + + @staticmethod + def key(volume_id: str) -> str: + return KEY_PREFIX + hashlib.sha1(volume_id.encode()).hexdigest()[:12] + + def close(self) -> None: + self.client.close() + self.backend.close() + + # -- desired state ------------------------------------------------------ + + def desired(self) -> tuple[dict[str, LioVolume], dict[str, dict[int, str]]]: + volumes: dict[str, LioVolume] = {} + acls: dict[str, dict[int, str]] = {iqn: {} for iqn in self.allow} + for iqn in self.fake.lio_all_iqns(): + acls.setdefault(iqn, {}) + for vid in self.fake.lio_volume_ids(): + vol = self.client.get_volume(vid) + if vol is None: + continue + key = self.key(vol.id) + volumes[key] = LioVolume(vol.size_bytes, _host_identity(vol)) + for m in self.client.list_mappings(vol.id): + for iqn in self.fake.lio_host_iqns(m.host_id): + luns = acls.setdefault(iqn, {}) + lun = m.lun if m.lun is not None and m.lun not in luns else _free(luns) + luns[lun] = key + return volumes, acls + + # -- reconcile ---------------------------------------------------------- + + def reconcile(self) -> None: + with self._lock: + want_vols, want_acls = self.desired() + have_vols = self.backend.list_volumes() + have_acls = self.backend.list_acls() + + # Unmap first: what is fenced is fenced before anything else moves. + for iqn, luns in have_acls.items(): + want = want_acls.get(iqn, {}) + for n, key in luns.items(): + if want.get(n) != key: + self.backend.unmap_lun(iqn, n) + if iqn not in want_acls: + self.backend.delete_acl(iqn) + for key in have_vols.keys() - want_vols.keys(): + self.backend.delete_volume(key) + for key, vol in want_vols.items(): + have = have_vols.get(key) + if have is None: + self.backend.create_volume(key, vol.size_bytes, vol.serial) + elif vol.size_bytes > have.size_bytes: + self.backend.resize_volume(key, vol.size_bytes) + have_acls = self.backend.list_acls() + for iqn, luns in want_acls.items(): + self.backend.ensure_acl(iqn) + current = have_acls.get(iqn, {}) + for n, key in luns.items(): + if current.get(n) != key: + self.backend.map_lun(iqn, n, key) + + +def _free(luns: dict[int, str]) -> int: + n = 0 + while n in luns: + n += 1 + return n + + +# -- kernel target -------------------------------------------------------------- + + +class ConfigfsLio: + """The kernel's iSCSI target through rtslib. One target, one portal group, + one ACL per initiator; volumes are sparse files on loop devices.""" + + def __init__(self, target_iqn: str, data_dir: str, portal: tuple[str, int] = ("0.0.0.0", 3260)): + # rtslib keeps its saved configuration here, and refuses to start on a + # host where no target ever ran and the directory is missing. + os.makedirs("/etc/rtslib-fb-target", exist_ok=True) + # python3-rtslib-fb, on the mock's host only + import rtslib_fb as rts # type: ignore[import-not-found] + + self.rts = rts + self.data_dir = data_dir + os.makedirs(data_dir, exist_ok=True) + fabric = rts.FabricModule("iscsi") + self._remove_previous(fabric, target_iqn) + self.target = rts.Target(fabric, target_iqn) + self.tpg = rts.TPG(self.target, 1) + for name, value in (("authentication", "0"), ("generate_node_acls", "0"), + ("demo_mode_write_protect", "0"), ("demo_mode_discovery", "1")): + self.tpg.set_attribute(name, value) + rts.NetworkPortal(self.tpg, portal[0], portal[1]) + self.tpg.enable = True + + def _remove_previous(self, fabric, target_iqn: str) -> None: + for target in list(fabric.targets): + if target.wwn == target_iqn: + target.delete() + for so in list(self.rts.RTSRoot().storage_objects): + if so.name.startswith(KEY_PREFIX): + self._delete_object(so) + + def _path(self, key: str) -> str: + return os.path.join(self.data_dir, key + ".img") + + def _object(self, key: str): + for so in self.rts.RTSRoot().storage_objects: + if so.name == key: + return so + raise KeyError(key) + + def _tpg_lun(self, key: str): + for lun in self.tpg.luns: + if lun.storage_object.name == key: + return lun + raise KeyError(key) + + def _delete_object(self, so) -> None: + dev = getattr(so, "udev_path", "") or "" + so.delete() + if dev.startswith("/dev/loop"): + subprocess.run(["losetup", "-d", dev], check=False) + path = self._path(so.name) + if os.path.exists(path): + os.unlink(path) + + def list_volumes(self) -> dict[str, LioVolume]: + out = {} + for so in self.rts.RTSRoot().storage_objects: + if so.name.startswith(KEY_PREFIX): + out[so.name] = LioVolume(os.path.getsize(self._path(so.name)), so.wwn) + return out + + def create_volume(self, key: str, size_bytes: int, serial: str) -> None: + path = self._path(key) + with open(path, "ab") as f: + f.truncate(size_bytes) + dev = subprocess.run(["losetup", "--find", "--show", "--direct-io=on", path], + check=True, capture_output=True, text=True).stdout.strip() + so = self.rts.BlockStorageObject(key, dev=dev, wwn=serial) + self.rts.LUN(self.tpg, storage_object=so) + logger.info("LIO: volume %s (%d bytes, serial %s) on %s", key, size_bytes, serial, dev) + + def resize_volume(self, key: str, size_bytes: int) -> None: + so = self._object(key) + with open(self._path(key), "ab") as f: + f.truncate(size_bytes) + subprocess.run(["losetup", "-c", so.udev_path], check=True) + logger.info("LIO: volume %s grown to %d bytes", key, size_bytes) + + def delete_volume(self, key: str) -> None: + self._tpg_lun(key).delete() + self._delete_object(self._object(key)) + logger.info("LIO: volume %s deleted", key) + + def list_acls(self) -> dict[str, dict[int, str]]: + return {acl.node_wwn: {ml.mapped_lun: ml.tpg_lun.storage_object.name for ml in acl.mapped_luns} + for acl in self.tpg.node_acls} + + def _acl(self, iqn: str): + return self.rts.NodeACL(self.tpg, iqn, mode="lookup") + + def ensure_acl(self, iqn: str) -> None: + self.rts.NodeACL(self.tpg, iqn) + + def delete_acl(self, iqn: str) -> None: + self._acl(iqn).delete() + + def map_lun(self, iqn: str, mapped_lun: int, key: str) -> None: + self.rts.MappedLUN(self._acl(iqn), mapped_lun, self._tpg_lun(key)) + logger.info("LIO: %s mapped to %s as LUN %d", key, iqn, mapped_lun) + + def unmap_lun(self, iqn: str, mapped_lun: int) -> None: + for ml in self._acl(iqn).mapped_luns: + if ml.mapped_lun == mapped_lun: + ml.delete() + logger.info("LIO: LUN %d of %s unmapped", mapped_lun, iqn) + + def close(self) -> None: + pass diff --git a/simplyblock_core/storage_arrays/testing/lio_selftest.py b/simplyblock_core/storage_arrays/testing/lio_selftest.py new file mode 100644 index 0000000000..d0136c620b --- /dev/null +++ b/simplyblock_core/storage_arrays/testing/lio_selftest.py @@ -0,0 +1,265 @@ +"""End-to-end self-test of an array plug-in against a mock array with a real +data path, on one Linux host. + +The mock array (a vendor fake with its LIO mirror, ``testing.lio``) is served +over HTTPS, and two iSCSI interfaces of this host, with two initiator names, +play two storage nodes. The array plug-in of the multi-attach mode +(``simplyblock_core.array.san``) drives the mock exactly as the control plane +does, and the node agent's own functions (``scan_for_new_luns``, +``find_lun_devices``, ``rescan_block_device``) find the LUNs on the host. The +checks are about the data path, on real block devices: + +1. a LUN created and attached to host A appears on A after a scan; +2. attached to host B as well, B reads the data A wrote (multi-attach); +3. fenced from A, A's I/O fails while B's goes on; +4. grown, the new size is seen by B after a rescan; +5. unfenced, A sees the LUN again; +6. deleted, it is gone from the target. + +Run as root on a host with LIO (``target_core_iblock``, ``iscsi_target_mod``), +open-iscsi, and python3-rtslib-fb:: + + python -m simplyblock_core.storage_arrays.testing.lio_selftest --vendor pure-flasharray + +On WSL, the portal goes on a dummy interface, because WSL's loopback refuses +the kernel target's listener:: + + ... lio_selftest --vendor pure-flasharray --portal 192.168.250.1:3260 --dummy-portal +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib +import os +import subprocess +import sys +import threading +import time + +from ..registry import ADAPTERS, canonical_vendor +from . import FakeTransport, serve +from .lio import ConfigfsLio, LioMirror + +GIB = 1 << 30 +TARGET = "iqn.2026-11.io.simplyblock:selftest" +IQN_A = "iqn.2026-11.io.simplyblock:selftest-host-a" +IQN_B = "iqn.2026-11.io.simplyblock:selftest-host-b" +PATTERN_BYTES = 4 << 20 + + +def sh(*cmd: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(list(cmd), check=check, capture_output=True, text=True) + + +def step(text: str) -> None: + print(f"--- {text}", flush=True) + + +class Hosts: + """Two iSCSI interfaces of this host, logged in to the mock's target.""" + + def __init__(self, portal: str): + self.portal = portal + if sh("pgrep", "-x", "iscsid", check=False).returncode != 0: + # Socket-activated under systemd; started by hand elsewhere. + if sh("systemctl", "start", "iscsid", check=False).returncode != 0: + subprocess.run(["iscsid"], check=True) + time.sleep(1) + for iface, iqn in (("sbtest-a", IQN_A), ("sbtest-b", IQN_B)): + sh("iscsiadm", "-m", "iface", "-I", iface, "-o", "new", check=False) + sh("iscsiadm", "-m", "iface", "-I", iface, "-o", "update", "-n", "iface.initiatorname", "-v", iqn) + sh("iscsiadm", "-m", "discovery", "-t", "st", "-p", portal, "-I", "sbtest-a", "-I", "sbtest-b") + sh("iscsiadm", "-m", "node", "-T", TARGET, "-p", portal, "--login") + + def sid_of(self, iqn: str) -> str: + """The SCSI host (``hostN``) of the session with initiator ``iqn``.""" + for sess in os.listdir("/sys/class/iscsi_session"): + base = f"/sys/class/iscsi_session/{sess}" + if open(f"{base}/initiatorname").read().strip() == iqn: + # .../hostN/sessionM: the session's SCSI host is in its path. + path = os.path.realpath(f"{base}/device") + return next(p for p in path.split("/") if p.startswith("host")) + raise RuntimeError(f"no iSCSI session of {iqn}") + + def close(self) -> None: + sh("iscsiadm", "-m", "node", "-T", TARGET, "-p", self.portal, "--logout", check=False) + sh("iscsiadm", "-m", "node", "-T", TARGET, "-p", self.portal, "-o", "delete", check=False) + for iface in ("sbtest-a", "sbtest-b"): + sh("iscsiadm", "-m", "iface", "-I", iface, "-o", "delete", check=False) + + +def device_of(node_utils, wwn: str, scsi_host: str, timeout: float = 20.0) -> str | None: + """The block device of LUN ``wwn`` behind ``scsi_host``, after scans.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + node_utils.scan_for_new_luns() + subprocess.run(["udevadm", "settle"], check=False) + for d in node_utils.find_lun_devices(wwn): + link = os.path.realpath(f"/sys/block/{d['name']}/device") + if f"/{scsi_host}/" in link: + return d["device_path"] + time.sleep(1) + return None + + +def write_pattern(dev: str) -> str: + flags = os.O_WRONLY + if sys.platform == "linux": + flags |= os.O_DIRECT | os.O_SYNC + data = os.urandom(PATTERN_BYTES) + fd = os.open(dev, flags) + try: + import mmap + buf = mmap.mmap(-1, PATTERN_BYTES) + buf.write(data) + os.write(fd, buf) + finally: + os.close(fd) + return hashlib.sha256(data).hexdigest() + + +def read_pattern(dev: str) -> str: + r = subprocess.run(["dd", f"if={dev}", "bs=1M", f"count={PATTERN_BYTES >> 20}", "iflag=direct", + "status=none"], capture_output=True) + if r.returncode != 0 or len(r.stdout) != PATTERN_BYTES: + raise OSError(r.stderr.decode(errors="replace").strip() or f"short read of {dev}") + return hashlib.sha256(r.stdout).hexdigest() + + +MODULES = ("configfs", "target_core_mod", "iscsi_target_mod", "target_core_iblock", "iscsi_tcp", "loop") + + +def prepare_host(portal_ip: str, dummy_portal: bool) -> None: + """Load the target and initiator modules and mount configfs (all + idempotent). With ``dummy_portal`` the portal address is put on a dummy + interface: hosts whose loopback refuses kernel listeners (WSL's mirrored + networking) need it.""" + for module in MODULES: + sh("modprobe", module) + if sh("mountpoint", "-q", "/sys/kernel/config", check=False).returncode != 0: + sh("mount", "-t", "configfs", "none", "/sys/kernel/config") + if dummy_portal and sh("ip", "link", "show", "sbdummy", check=False).returncode != 0: + sh("ip", "link", "add", "sbdummy", "type", "dummy") + sh("ip", "addr", "add", f"{portal_ip}/32", "dev", "sbdummy") + sh("ip", "link", "set", "sbdummy", "up") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--vendor", default="pure-flasharray", help=", ".join(sorted(ADAPTERS))) + parser.add_argument("--port", type=int, default=8443) + parser.add_argument("--portal", default="127.0.0.1:3260") + parser.add_argument("--lio-dir", default="/var/tmp/sb-lio-selftest") + parser.add_argument("--dummy-portal", action="store_true", + help="put the portal address on a dummy interface (WSL)") + args = parser.parse_args() + + if sys.platform != "linux": + print("runs on Linux only", file=sys.stderr) + return 2 + if os.geteuid() != 0: + print("run as root", file=sys.stderr) + return 2 + from simplyblock_core import array + from simplyblock_web import node_utils + + vendor = canonical_vendor(args.vendor) + module = importlib.import_module(f"{__package__}.{ADAPTERS[vendor].split(':')[0].split('.')[-1]}") + setup = module.contract_setup() + ip, _, port = args.portal.rpartition(":") + prepare_host(ip, args.dummy_portal) + backend = ConfigfsLio(TARGET, args.lio_dir, (ip, int(port))) + mirror = LioMirror(setup.fake, setup.config, setup.credential, backend, + transport=FakeTransport(setup.fake), allow=[IQN_A, IQN_B]) + mirror.reconcile() + server = serve(setup.fake, "127.0.0.1", args.port, after_request=mirror.reconcile) + threading.Thread(target=server.serve_forever, daemon=True).start() + + hosts = Hosts(args.portal) + failures: list[str] = [] + + def check(ok: bool, what: str) -> None: + print(("PASS " if ok else "FAIL ") + what, flush=True) + if not ok: + failures.append(what) + + plugin = None + lun = None + try: + endpoint = f"https://127.0.0.1:{args.port}" + options = dict(setup.config.options, protocol="iscsi", ca_bundle=open(server.certfile).read()) + plugin = array.plugin_class(vendor)(endpoint=endpoint, options=options, cluster_id="selftest") + plugin.connect(endpoint, {n: setup.credential.get(n) for n in setup.credential.field_names}, options) + host_a = plugin.ensure_host("selftest-a", {"iscsi_iqn": IQN_A}) + host_b = plugin.ensure_host("selftest-b", {"iscsi_iqn": IQN_B}) + scsi_a, scsi_b = hosts.sid_of(IQN_A), hosts.sid_of(IQN_B) + + step("1. create and attach to host A") + lun = plugin.create_lun("selftest-lun", GIB, {}) + plugin.attach(lun["id"], host_a) + dev_a = device_of(node_utils, lun["wwn"], scsi_a) + check(dev_a is not None, f"LUN {lun['wwn']} appears on host A ({dev_a})") + if dev_a is None: + return 1 + digest = write_pattern(dev_a) + + step("2. attach to host B as well (multi-attach)") + plugin.attach(lun["id"], host_b) + dev_b = device_of(node_utils, lun["wwn"], scsi_b) + check(dev_b is not None, f"LUN appears on host B ({dev_b})") + if dev_b is None: + return 1 + check(read_pattern(dev_b) == digest, "host B reads what host A wrote") + check(sorted(plugin.get(lun["id"])["hosts"]) == sorted([host_a, host_b]), + "the array reports both hosts") + + step("3. fence host A") + plugin.fence(lun["id"], host_a) + try: + read_pattern(dev_a) + check(False, "host A's I/O fails after the fence") + except OSError as e: + check(True, f"host A's I/O fails after the fence ({str(e)[:60]})") + check(read_pattern(dev_b) == digest, "host B still reads the data") + + step("4. grow") + plugin.resize(lun["id"], 2 * GIB) + grown = node_utils.rescan_block_device(dev_b) + check(grown["size"] >= 2 * GIB, f"host B sees {grown['size']} bytes after a rescan") + + step("5. unfence host A") + plugin.unfence(lun["id"], host_a) + sh("iscsiadm", "-m", "session", "--rescan", check=False) + dev_a2 = device_of(node_utils, lun["wwn"], scsi_a) + check(dev_a2 is not None and read_pattern(dev_a2) == digest, "host A reads the data again") + + step("6. delete") + plugin.delete_lun(lun["id"]) + check(mirror.key(lun["id"]) not in backend.list_volumes(), "the LUN is gone from the target") + lun = None + finally: + if plugin is not None and lun is not None: + try: + plugin.delete_lun(lun["id"]) + except Exception as e: # noqa: BLE001 + print(f"cleanup: {e}") + hosts.close() + server.shutdown() + backend_cleanup(backend) + print(f"{'FAILED' if failures else 'PASSED'}: {len(failures)} failure(s)", flush=True) + return 1 if failures else 0 + + +def backend_cleanup(backend: ConfigfsLio) -> None: + for key in list(backend.list_volumes()): + for iqn, luns in backend.list_acls().items(): + for n, k in luns.items(): + if k == key: + backend.unmap_lun(iqn, n) + backend.delete_volume(key) + backend.target.delete() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/simplyblock_core/storage_arrays/testing/netapp_ontap.py b/simplyblock_core/storage_arrays/testing/netapp_ontap.py index 735f31b498..b6ea286626 100644 --- a/simplyblock_core/storage_arrays/testing/netapp_ontap.py +++ b/simplyblock_core/storage_arrays/testing/netapp_ontap.py @@ -268,6 +268,20 @@ def _job(self, description: str, action: Callable[[], None]) -> Reply: href = f"/api/cluster/jobs/{jid}" return Reply(202, {"job": {"uuid": jid, "_links": {"self": {"href": href}}}}) + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return list(self.luns) + list(self.namespaces) + + def lio_all_iqns(self) -> list[str]: + return sorted({i["name"].lower() for g in self.igroups.values() for i in g["initiators"] if i["name"].lower().startswith("iqn.")}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + for g in self.igroups.values(): + if host_id in (g["uuid"], g["name"]): + return [i["name"].lower() for i in g["initiators"] if i["name"].lower().startswith("iqn.")] + return [] + @route("GET", r"/api/cluster/jobs/" + _U) def get_job(self, req, uuid): self._auth(req) diff --git a/simplyblock_core/storage_arrays/testing/pure.py b/simplyblock_core/storage_arrays/testing/pure.py index fc79564cd9..bc6e417038 100644 --- a/simplyblock_core/storage_arrays/testing/pure.py +++ b/simplyblock_core/storage_arrays/testing/pure.py @@ -55,6 +55,18 @@ def _auth(self, req: Request) -> None: if req.headers.get("x-auth-token") not in self.sessions: raise _err("Authentication failed.", 401) + + # -- LIO data path (testing.lio) ----------------------------------------- + def lio_volume_ids(self) -> list[str]: + return [v["id"] for v in self.volumes.values() if not v.get("destroyed")] + + def lio_all_iqns(self) -> list[str]: + return sorted({str(i).lower() for h in self.hosts.values() for i in h.get("iqns", [])}) + + def lio_host_iqns(self, host_id: str) -> list[str]: + host = self.hosts.get(host_id) + return [str(i).lower() for i in (host or {}).get("iqns", [])] + @route("GET", "/api/api_version") def api_version(self, req): return Reply(200, {"version": self.versions}) diff --git a/tests/unit/storage_arrays/test_lio.py b/tests/unit/storage_arrays/test_lio.py new file mode 100644 index 0000000000..9d92107c02 --- /dev/null +++ b/tests/unit/storage_arrays/test_lio.py @@ -0,0 +1,105 @@ +"""The LIO data path of the mock arrays (storage_arrays.testing.lio): a fake's +volumes, hosts, and mappings mirrored into an iSCSI target, here into the +in-memory backend. Every vendor fake with iSCSI runs the same scenario through +its own adapter, which is what the control plane would send it.""" +from __future__ import annotations + +import pytest + +from simplyblock_core.array.san import host_wwn +from simplyblock_core.storage_arrays.base import HostSpec, Initiator, Protocol +from simplyblock_core.storage_arrays.testing import FakeTransport +from simplyblock_core.storage_arrays.testing.lio import LioMirror, MemoryLio + +from .conftest import VENDORS, Harness, fake_module + +GIB = 1 << 30 +ISCSI_VENDORS = [v for v in VENDORS if v != "dell-powerflex"] +UUID = "6f1d3a52-8c4b-4e0a-9d27-3b8e4c1f7a90" + + +class Bed: + def __init__(self, vendor: str): + self.setup = fake_module(vendor).contract_setup() + self.harness = Harness(self.setup) + self.client = self.harness.client + self.backend = MemoryLio() + self.mirror = LioMirror(self.setup.fake, self.setup.config, self.setup.credential, + self.backend, transport=FakeTransport(self.setup.fake), + session_kwargs={"sleep": lambda s: None}) + groups = self.setup.initiators[Protocol.ISCSI] + self.iqn_a, self.iqn_b = groups[0][0].address.lower(), groups[1][0].address.lower() + + def host(self, name: str, iqn: str) -> str: + return self.client.ensure_host(HostSpec( + name=name, initiators=(Initiator(protocol=Protocol.ISCSI, address=iqn),))) + + +@pytest.fixture(params=ISCSI_VENDORS) +def bed(request) -> Bed: + b = Bed(request.param) + yield b + b.client.close() + b.mirror.close() + + +def test_volumes_hosts_and_mappings_reach_the_target(bed): + vol = bed.client.create_volume(UUID, 10 * GIB) + host_a = bed.host("sbtest-a", bed.iqn_a) + bed.mirror.reconcile() + lun = bed.backend.volumes[bed.mirror.key(vol.id)] + assert lun.size_bytes == vol.size_bytes + # The host sees the identity the plug-in reports for the LUN. + assert lun.serial == host_wwn(vol, Protocol.ISCSI) + # A registered host can log in, and sees nothing yet. + assert bed.backend.acls[bed.iqn_a] == {} + + bed.client.map_volume(vol.id, host_a) + bed.mirror.reconcile() + (mapped,) = bed.backend.acls[bed.iqn_a].values() + assert mapped == bed.mirror.key(vol.id) + + # Multi-attach: a second host sees the same LUN. + host_b = bed.host("sbtest-b", bed.iqn_b) + bed.client.map_volume(vol.id, host_b) + bed.mirror.reconcile() + assert list(bed.backend.acls[bed.iqn_b].values()) == [bed.mirror.key(vol.id)] + + # Fence: the unmapped host loses the LUN, the other keeps it. + bed.client.unmap_volume(vol.id, host_a) + bed.mirror.reconcile() + assert bed.backend.acls[bed.iqn_a] == {} + assert list(bed.backend.acls[bed.iqn_b].values()) == [bed.mirror.key(vol.id)] + + grown = bed.client.resize_volume(vol.id, 20 * GIB) + bed.mirror.reconcile() + assert bed.backend.volumes[bed.mirror.key(vol.id)].size_bytes == grown.size_bytes + + bed.client.delete_volume(vol.id) + bed.mirror.reconcile() + assert bed.mirror.key(vol.id) not in bed.backend.volumes + assert bed.backend.acls[bed.iqn_b] == {} + + +def test_reconcile_is_idempotent(bed): + vol = bed.client.create_volume(UUID, 10 * GIB) + bed.client.map_volume(vol.id, bed.host("sbtest-a", bed.iqn_a)) + bed.mirror.reconcile() + before = bed.backend.ops + bed.mirror.reconcile() + assert bed.backend.ops == before + + +def test_allowed_initiators_can_log_in_before_they_are_registered(): + b = Bed("pure-flasharray") + b.mirror.allow = ["iqn.2026-01.io.sb:not-registered"] + b.mirror.reconcile() + assert b.backend.acls["iqn.2026-01.io.sb:not-registered"] == {} + + +def test_lun_numbers_follow_the_array(): + b = Bed("infinidat-infinibox") # reports a LUN number per mapping + vol = b.client.create_volume(UUID, 10 * GIB) + m = b.client.map_volume(vol.id, b.host("sbtest-a", b.iqn_a)) + b.mirror.reconcile() + assert list(b.backend.acls[b.iqn_a]) == [m.lun] From 44b4cd7f34512d739b5c5ceb396de43435910bd3 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 19:35:07 +0100 Subject: [PATCH 37/50] multi-attach takeover: a dead standby is left for the next one 2026-11-20 FTT2 bed, dual reboot sn2 + hard reset sn3: the standby chosen for sn3's LUN was sn2, whose reboot had just begun (its agent still answered the path probe). Every RPC to it then hung on retries (~28 s), the reservation fence failed, the array fence followed and the export was retried on the same dead node. The takeover failed although a third host could read the LUN; the LUN went unavailable and the 1+0 cluster suspended. - An export that fails on a standby is retried there once only if its SPDK still answers; otherwise the next standby candidate is chosen (excluding those tried) until the deadline. - The activation (role) polls use a 1 s client without retries, so a dead standby shows within MULTI_ATTACH_ACTIVATION_WAIT_SEC. - No cleanup calls into a standby whose SPDK no longer answers. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/controllers/lun_takeover.py | 70 ++++++++++++++------ tests/unit/test_lun_takeover.py | 59 +++++++++++++++-- 2 files changed, 105 insertions(+), 24 deletions(-) diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index 7aa8b1b793..c676bce728 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -251,22 +251,37 @@ def _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) logger.error(msg) _event(cluster_id, dev, msg, EventObj.LEVEL_CRITICAL) return NO_STANDBY - _check_deadline(deadline, f"exporting the LUN on {standby.get_id()}") - - # 4. Open-only export on the standby. - try: - export = _export_on_standby(db, cluster, standby, dev, path, deadline) - except TakeoverError as e: - if not by_reservation: - raise - # The reservation was not taken: fence at the array, then once more. - logger.warning("LUN takeover of %s: fence by reservation failed (%s); fencing " - "at the array", device_id, e) - by_reservation = False - _fence(cluster, dev, failed) - _record_fenced(db, owner, device_id, failed_node_id) - _check_deadline(deadline, f"exporting the LUN on {standby.get_id()} again") - export = _export_on_standby(db, cluster, standby, dev, path, deadline) + # 4. Open-only export on the standby. A standby that fails and no + # longer answers is left for the next candidate (2026-11-20 FTT2 bed: + # the standby chosen was rebooting; the takeover failed although a + # third node could read the LUN, and the 1+0 cluster suspended). + tried, retried = set(), set() + while True: + _check_deadline(deadline, f"exporting the LUN on {standby.get_id()}") + try: + export = _export_on_standby(db, cluster, standby, dev, path, deadline) + break + except TakeoverError as e: + sid = standby.get_id() + if by_reservation: + # The reservation was not taken: fence at the array. + logger.warning("LUN takeover of %s: fence by reservation failed (%s); " + "fencing at the array", device_id, e) + by_reservation = False + _fence(cluster, dev, failed) + _record_fenced(db, owner, device_id, failed_node_id) + if sid not in retried and _reachable(standby): + retried.add(sid) + continue + tried.add(sid) + logger.warning("LUN takeover of %s: export on standby %s failed (%s); trying " + "another standby", device_id, sid, e) + _check_deadline(deadline, "choosing another standby") + standby, path, why = _choose_standby(owner, dev, failed_node_id, nodes_by_id, + deadline, exclude=tried) + if standby is None: + raise TakeoverError(f"export failed on {sorted(tried)} and no other standby " + f"can read the LUN ({why}): {e}") if by_reservation: # Fenced by the reservation: recorded now, detached at the array # in the background (the attachment reconcile detaches it too). @@ -437,11 +452,11 @@ def _mut(n): # --------------------------------------------------------- 2 + 3. standby -def _choose_standby(owner, dev, failed_node_id, nodes_by_id, deadline): +def _choose_standby(owner, dev, failed_node_id, nodes_by_id, deadline, exclude=()): reasons = [] for nid in multi_attach.standby_candidates(owner, failed_node_id): node = nodes_by_id.get(nid) - if node is None: + if node is None or nid in exclude: continue if node.status != StorageNode.STATUS_ONLINE: reasons.append(f"{nid}: {node.status}") @@ -463,6 +478,15 @@ def _choose_standby(owner, dev, failed_node_id, nodes_by_id, deadline): return None, None, "; ".join(reasons) or "no standby candidate" +def _reachable(node) -> bool: + """Whether the node's SPDK answers right now (no retries).""" + try: + node.rpc_client(timeout=constants.MULTI_ATTACH_PROBE_TIMEOUT_SEC, retry=0).get_version() + return True + except Exception: # noqa: BLE001 + return False + + def _probe(node, dev, deadline): try: result, err = node.client(timeout=_remaining(deadline, 15)).lun_info(dev.lun_wwn) @@ -550,7 +574,7 @@ def _export_on_standby(db, cluster, standby, dev, path, deadline) -> dict: created.append(("alceml", alceml)) if kwargs.get("reservation_key"): # Opened active with a key: it preempts the old holder (fence). - _wait_active(rpc, alceml, deadline) + _wait_active(standby.rpc_client(timeout=1, retry=0), alceml, deadline) try: from simplyblock_core.controllers import qos_controller rpc.alceml_set_qos_weights(qos_controller.get_qos_weights_list(cluster.get_id())) @@ -590,7 +614,13 @@ def _export_on_standby(db, cluster, standby, dev, path, deadline) -> dict: nguid=multi_attach.namespace_nguid(dev)): raise TakeoverError(f"namespace {pt} not added to {nqn} on {sid}") except Exception: - _teardown(rpc, sid, created) + if created and not _reachable(standby): + # Its SPDK is gone: what it held went with it (each call would + # only wait out the retries). + logger.warning("LUN takeover cleanup on %s skipped: SPDK not answering (%s)", + sid, [name for _, name in created]) + else: + _teardown(rpc, sid, created) raise aliases = (pt_info or {}).get("aliases") or [] return { diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index 1e07c6c99f..dcaf6b198c 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -45,6 +45,12 @@ def __init__(self, name): self.subsystems = {} self.calls = [] self.fail = set() + self.down = False + + def get_version(self): + if self.down: + raise ConnectionError("Could not reach remote") + return {"version": "fake"} def _rec(self, op, *args, **kwargs): self.calls.append((op, args, kwargs)) @@ -375,17 +381,29 @@ def test_rerun_is_idempotent(self): def test_rerun_after_partial_export_reuses_it(self): db, _ = _bed() - b = db.nodes["node-b"] - b.spdk.fail.add("nvmf_subsystem_add_ns") + standbys = [db.nodes[n] for n in ("node-b", "node-c", "node-d")] + for n in standbys: + n.spdk.fail.add("nvmf_subsystem_add_ns") self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) - # Cleanup closed what this attempt opened; nothing recorded. + # Cleanup closed what each attempt opened; nothing recorded. self.assertEqual(self.dev(db).serving_node(), "node-a") + b = standbys[0] self.assertNotIn(f"alceml_{DEV_ID}", b.spdk.bdevs) self.assertTrue(b.spdk.ops("bdev_alceml_delete")) - b.spdk.fail.clear() + for n in standbys: + n.spdk.fail.clear() self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) self.assertEqual(self.dev(db).serving_node(), "node-b") + def test_failed_export_moves_to_the_next_standby(self): + db, _ = _bed() + db.nodes["node-b"].spdk.fail.add("nvmf_subsystem_add_ns") + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + # node-b was retried once (it answers), then left; its stack closed. + self.assertEqual(len(db.nodes["node-b"].spdk.ops("bdev_alceml_create")), 2) + self.assertNotIn(f"alceml_{DEV_ID}", db.nodes["node-b"].spdk.bdevs) + def test_standby_is_chosen_while_the_fence_runs(self): import threading db, _ = _bed() @@ -1070,6 +1088,39 @@ def test_failed_activation_falls_back_to_the_array_fence(self): self.assertEqual(len(b.spdk.ops("bdev_alceml_delete")), 1) self.assertEqual(self.dev(db).serving_node(), "node-b") + def test_dead_standby_is_left_for_the_next_one(self): + # 2026-11-20 FTT2 bed, dual reboot sn2 + hard reset sn3: the standby + # chosen was rebooting; the LUN must still be taken over by another. + db, _ = self._bed() + b, c = db.nodes["node-b"], db.nodes["node-c"] + + def b_dies(name): + b.spdk.down = True + return {"error": "Could not reach remote"} + b.spdk.alceml_get_role = b_dies + self._roles(c, {"role": "active", "state": "active", "epoch": 3}) + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 0.2): + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + self.assertIn("node-a", self.dev(db).fenced_node_ids) + # The array fence ran (the reservation was not taken on node-b). + self.assertIn(("fence", WWN, "host-node-a"), FakeArrayPlugin.calls) + # No cleanup calls into the dead standby. + self.assertEqual(b.spdk.ops("bdev_alceml_delete"), []) + + def test_no_standby_left_fails(self): + db, _ = self._bed() + def dies(spdk): + def role(name): + spdk.down = True + return {"error": "gone"} + return role + for nid in ("node-b", "node-c", "node-d"): + db.nodes[nid].spdk.alceml_get_role = dies(db.nodes[nid].spdk) + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 0.2): + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.assertEqual(self.dev(db).serving_node(), "node-a") + def test_without_reservations_the_array_fences_first(self): db, _ = _bed() self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) From 3e921e02e94eb2f2c8bc669272d9d15124361fe5 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 20:14:58 +0100 Subject: [PATCH 38/50] multi-attach: port fences are not node failures; watchdog; SPDK crash logs 2026-11-20 FTT2 bed, hard reset of sn3: one unavailable LUN made SPDK fence the client port of every LVS of the 1+0 cluster; within 4 s the monitor set three healthy nodes DOWN (every liveness probe passing), the node floor broke and the cluster suspended. Multi-attach clusters only; regular clusters unchanged: - storage_node_monitor: a node's own LVS port that SPDK reports blocked gets the fence grace peer-owned ports have (PEER_PORT_FENCE_GRACE_SEC) instead of the 1 s re-read, so the stale-fence remediation can act first. - storage_node_monitor: more nodes than the cluster tolerates failing only their port check within PORT_FAILURE_CORRELATION_SEC are not set DOWN; one cluster event (storage_events.snode_port_check_correlated) instead. - lun_watchdog: an "SPDK running" agent answer restarts the silence; it survived such answers, and one agent timeout read as "no answer for 6.1s". Node agent (docker): before a start/kill/cleanup removes an SPDK container that had already exited on its own, its last 30000 log lines and exit state are kept in /etc/simplyblock/spdk-crash-logs (newest 10). The three SPDK deaths of that cascade left no trace. Co-Authored-By: Claude Opus 5.5 --- .../controllers/storage_events.py | 14 +++ simplyblock_core/services/lun_watchdog.py | 9 +- .../services/storage_node_monitor.py | 76 +++++++++++- .../api/internal/storage_node/docker.py | 42 +++++++ tests/unit/test_lun_watchdog.py | 14 +++ tests/unit/test_multi_attach_port_check.py | 110 ++++++++++++++++++ tests/unit/test_spdk_crash_logs.py | 58 +++++++++ 7 files changed, 317 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_multi_attach_port_check.py create mode 100644 tests/unit/test_spdk_crash_logs.py diff --git a/simplyblock_core/controllers/storage_events.py b/simplyblock_core/controllers/storage_events.py index e83ddbb696..30a2aa4125 100644 --- a/simplyblock_core/controllers/storage_events.py +++ b/simplyblock_core/controllers/storage_events.py @@ -144,6 +144,20 @@ def snode_rpc_timeout(node, timeout_seconds, caused_by=ec.CAUSED_BY_MONITOR): node_id=node.get_id()) +def snode_port_check_correlated(node, reason, caused_by=ec.CAUSED_BY_MONITOR): + """Multi-attach clusters: more nodes than the cluster tolerates fail only + their port check at once; none of them is set DOWN for it.""" + ec.log_event_cluster( + cluster_id=node.cluster_id, + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_STATUS_CHANGE, + db_object=node, + caused_by=caused_by, + event_level=EventObj.LEVEL_ERROR, + message=f"Port checks failing cluster-wide, nodes not set down: {reason}", + node_id=node.get_id()) + + def snode_orphaned_objects(node, orphans, caused_by=ec.CAUSED_BY_MONITOR): """Raised by the lvol monitor's orphan sweep: lvstore objects exist on this node with no record in the database. Detect-only — see diff --git a/simplyblock_core/services/lun_watchdog.py b/simplyblock_core/services/lun_watchdog.py index 9cbac1423c..8d9f7756b4 100644 --- a/simplyblock_core/services/lun_watchdog.py +++ b/simplyblock_core/services/lun_watchdog.py @@ -190,10 +190,15 @@ def record(self, node, answer, now, sent=None) -> str | None: nid, now - t.first_fail) t.first_fail, t.fired_at, t.last_ok = None, None, now return None - if t.first_fail is None: - t.first_fail = now if sent is None else sent if answer == SPDK_UP: + # The agent saw SPDK running: the node is there, its RPC is + # slow. The silence restarts at the next unanswered probe (it + # used to survive this answer, and one agent timeout after it + # read as "no answer for 6.1s", 2026-11-20 FTT2 bed). + t.first_fail = None return None + if t.first_fail is None: + t.first_fail = now if sent is None else sent if answer == SPDK_DOWN: return "SPDK not running" silent = now - t.first_fail diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index b99cd90d1f..d5bbf5dfcd 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -2357,6 +2357,60 @@ def node_rpc_timeout_check_and_report(node): return False +#: Node id -> (cluster id, monotonic time) of its last failed port check, for +#: the correlation guard below. Cleared by a passing check. +_port_failed_at: dict = {} +_port_failed_lock = threading.Lock() +#: A failed port check counts toward the correlation for this long: a few node +#: check cycles. +PORT_FAILURE_CORRELATION_SEC = 30.0 +_correlated_event_at: dict = {} + + +def _multi_attach_cluster(cluster_id) -> bool: + try: + return multi_attach.is_multi_attach(db.get_cluster_by_id(cluster_id)) + except Exception: # noqa: BLE001 + return False + + +def _note_port_check(snode, result): + with _port_failed_lock: + if result is False: + _port_failed_at[snode.get_id()] = (snode.cluster_id, time.monotonic()) + elif result is True: + _port_failed_at.pop(snode.get_id(), None) + + +def _correlated_port_failure(cluster, snode) -> str: + """Multi-attach clusters (only): why a failed port check of ``snode`` must + not set it DOWN, or "". Nodes that pass every liveness probe and fail only + the port check, more of them at once than the cluster tolerates, point at a + device or data-path condition (one bad LUN fences every LVS of a 1+0 + cluster), not at the nodes: setting them DOWN breaks the node floor and + suspends the cluster (2026-11-20 FTT2 bed). Same rule as the LUN + watchdog's observer guard.""" + if not multi_attach.is_multi_attach(cluster): + return "" + now = time.monotonic() + cid = cluster.get_id() + with _port_failed_lock: + failing = sorted(nid for nid, (c, t) in _port_failed_at.items() + if c == cid and now - t <= PORT_FAILURE_CORRELATION_SEC) + ftt = max(1, int(getattr(cluster, "max_fault_tolerance", 1) or 1)) + if len(failing) <= ftt: + return "" + reason = (f"{len(failing)} nodes fail only their port check at once ({failing}), more " + f"than the cluster tolerates ({ftt}): a data-path condition, not node failures") + if now - _correlated_event_at.get(cid, 0) > 60: + _correlated_event_at[cid] = now + try: + storage_events.snode_port_check_correlated(snode, reason) + except Exception as e: # noqa: BLE001 + logger.warning("Correlated port-check event not sent: %s", e) + return reason + + def node_port_check_fun(snode): node_port_check = True if snode.lvstore_status == "ready": @@ -2413,11 +2467,19 @@ def node_port_check_fun(snode): port_lvs_owner[_p] = snode.get_id() own_port = _p + # Multi-attach (1+0): a port SPDK reports blocked is its own fence + # (failed IO queued, leadership change), and one bad device fences + # every LVS of the cluster at once. The node's own port gets the + # grace peer-owned ports have, so the leak remediation can act before + # the block counts against the node (2026-11-20 FTT2 bed: three + # healthy nodes set DOWN within 4 s of such a fence). + own_port_grace = _multi_attach_cluster(snode.cluster_id) + # Batched: one nvmf_get_blocked_ports fetch answers every port, so # carrying the advisory ports costs no extra RPC. try: port_results = health_controller.check_ports_on_node(snode, ports) - if own_port is not None and port_results.get(own_port) is False: + if own_port is not None and port_results.get(own_port) is False and not own_port_grace: _reread_blocked_own_port(snode, own_port, port_results) for port, ret in port_results.items(): if port in advisory_ports: @@ -2426,13 +2488,14 @@ def node_port_check_fun(snode): f"(advisory: peer owns this LVS and may be fencing it)") continue owner_id = port_lvs_owner.get(port) - if ret is False and owner_id and owner_id != snode.get_id(): + if ret is False and owner_id and (owner_id != snode.get_id() or own_port_grace): held = _blocked_port_age(snode, port) if held < PEER_PORT_FENCE_GRACE_SEC: + whose = "peer-owned" if owner_id != snode.get_id() else "own" logger.info( f"Check: node port {snode.mgmt_ip}, {port} ... {ret} " - f"(peer-owned LVS fenced {held:.0f}s, within the " - f"{PEER_PORT_FENCE_GRACE_SEC:.0f}s takeover grace; not counted)") + f"({whose} LVS fenced {held:.0f}s, within the " + f"{PEER_PORT_FENCE_GRACE_SEC:.0f}s fence grace; not counted)") continue logger.info(f"Check: node port {snode.mgmt_ip}, {port} ... {ret}") node_port_check &= ret @@ -3163,9 +3226,14 @@ def check_node(snode): else: node_port_check = port_check_holder["result"] + _note_port_check(snode, node_port_check) if node_port_check is False: cluster = db.get_cluster_by_id(snode.cluster_id) if cluster.status in [Cluster.STATUS_ACTIVE, Cluster.STATUS_DEGRADED, Cluster.STATUS_READONLY]: + held = _correlated_port_failure(cluster, snode) + if held: + logger.error("Port check failed on %s, node not set down: %s", snode.get_id(), held) + return True logger.error("Port check failed") set_node_down(snode) return True diff --git a/simplyblock_web/api/internal/storage_node/docker.py b/simplyblock_web/api/internal/storage_node/docker.py index 98d4403b6f..80a70b317d 100644 --- a/simplyblock_web/api/internal/storage_node/docker.py +++ b/simplyblock_web/api/internal/storage_node/docker.py @@ -92,6 +92,45 @@ class SPDKParams(BaseModel): mcp_max_unavailable: int | None = Field(None) # OpenShift-only (MCP); ignored here +#: Where the agent keeps the last log lines of an SPDK container that died on +#: its own, before a restart removes it (host-mounted /etc/simplyblock). +SPDK_CRASH_LOG_DIR = os.environ.get("SPDK_CRASH_LOG_DIR", "/etc/simplyblock/spdk-crash-logs") +SPDK_CRASH_LOG_LINES = 30000 +SPDK_CRASH_LOG_KEEP = 10 + + +def _preserve_exited_spdk_logs(client, name): + """Save the log tail and exit state of ``name`` if its container already + exited -- SPDK died on its own -- before a restart removes it with its + logs (2026-11-20 FTT2 bed: three SPDKs died in a cascade, every trace of + why went with the containers). A running container (a deliberate kill) + is left alone; best-effort, never raises.""" + try: + container = client.containers.get(name) + state = (container.attrs or {}).get("State") or {} + if state.get("Status") not in ("exited", "dead"): + return None + os.makedirs(SPDK_CRASH_LOG_DIR, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + path = os.path.join(SPDK_CRASH_LOG_DIR, f"{name.strip('/')}-{stamp}.log") + logs = container.logs(tail=SPDK_CRASH_LOG_LINES, timestamps=True) + with open(path, "wb") as f: + exit_state = {k: state.get(k) for k in ( + "ExitCode", "OOMKilled", "Error", "StartedAt", "FinishedAt")} + f.write(json.dumps(exit_state).encode() + os.linesep.encode()) + f.write(logs if isinstance(logs, bytes) else str(logs).encode()) + kept = sorted(Path(SPDK_CRASH_LOG_DIR).glob("spdk_*.log"), key=lambda q: q.stat().st_mtime) + for old in kept[:-SPDK_CRASH_LOG_KEEP]: + old.unlink(missing_ok=True) + logger.warning("SPDK container %s had exited (code %s, OOM %s); its log kept in %s", + name, state.get("ExitCode"), state.get("OOMKilled"), path) + return path + except Exception as exc: # noqa: BLE001 + logger.info("no crash log kept for %s: %s", name, exc) + return None + + + @api.post('/spdk_process_start', responses={ 200: {'content': {'application/json': {'schema': utils.response_schema({ 'type': 'boolean' @@ -107,6 +146,7 @@ def spdk_process_start(body: SPDKParams): node_docker = get_docker_client(timeout=60 * 3) spdk_names = [f"/spdk_{body.rpc_port}", f"/spdk_proxy_{body.rpc_port}"] for name in spdk_names: + _preserve_exited_spdk_logs(node_docker, name) core_utils.remove_container(node_docker, name, graceful_timeout=0) # Confirm the previous instances are actually gone before we start a fresh @@ -263,6 +303,7 @@ def spdk_process_kill(query: utils.RPCPortParams): # SIGKILLing whatever currently answers to the name. targets = [] for name in names: + _preserve_exited_spdk_logs(client, name) try: targets.append(client.containers.get(name)) except NotFound: @@ -332,6 +373,7 @@ def spdk_process_cleanup(query: utils.RPCPortParams): names = [f"/spdk_{query.rpc_port}", f"/spdk_proxy_{query.rpc_port}"] ok = True for name in names: + _preserve_exited_spdk_logs(client, name) try: container = client.containers.get(name) except NotFound: diff --git a/tests/unit/test_lun_watchdog.py b/tests/unit/test_lun_watchdog.py index 17748a5872..5bf01580ee 100644 --- a/tests/unit/test_lun_watchdog.py +++ b/tests/unit/test_lun_watchdog.py @@ -127,6 +127,20 @@ def test_running_spdk_is_never_lost(self): for t in range(60): self.assertIsNone(self.w.record(self.a, wd.SPDK_UP, 100.0 + t)) + def test_spdk_running_restarts_the_silence(self): + # Slow RPC while the agent sees SPDK running, then one agent timeout: + # not "no answer for" the whole stretch. + self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 100.0)) + for t in (101.0, 102.0, 103.0, 104.0, 105.0): + self.assertIsNone(self.w.record(self.a, wd.SPDK_UP, t)) + self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 106.0, sent=105.2)) + self.assertTrue(self.w.record(self.a, wd.NO_ANSWER, 107.4, sent=106.0) + .startswith("no answer for 2.2s")) + + def test_slow_node_is_not_counted_as_failing(self): + self.w.record(self.a, wd.SPDK_UP, 100.0) + self.assertIsNone(self.w.tracks["node-a"].first_fail) + def test_an_answer_within_the_window_cancels(self): self.assertIsNone(self.w.record(self.a, wd.NO_ANSWER, 100.0)) self.assertIsNone(self.w.record(self.a, wd.OK, 100.0 + LOST - 0.5)) diff --git a/tests/unit/test_multi_attach_port_check.py b/tests/unit/test_multi_attach_port_check.py new file mode 100644 index 0000000000..390c0296df --- /dev/null +++ b/tests/unit/test_multi_attach_port_check.py @@ -0,0 +1,110 @@ +"""Port checks on multi-attach clusters (1+0): a port SPDK reports blocked is +its own fence, and one bad device fences every LVS of the cluster at once +(2026-11-20 FTT2 bed: a stale error event marked a taken-over LUN +unavailable; three healthy nodes were set DOWN within 4 s, the node floor +broke and the cluster suspended). + + A. the node's own port gets the fence grace peer-owned ports have; + B. more nodes than the cluster tolerates failing only the port check at + once are not set DOWN. +Regular clusters keep the 1 s re-read and set DOWN as before.""" + +import time +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.services import storage_node_monitor as snm +from tests.unit.test_port_check_deliberate_fence import OWN_PORT, _node + + +def _cluster(multi, ftt=2): + c = Cluster() + c.uuid = "c1" + c.status = Cluster.STATUS_ACTIVE + c.multi_attach = multi + c.max_fault_tolerance = ftt + return c + + +class TestOwnPortGrace(unittest.TestCase): + + def setUp(self): + snm._blocked_port_since.clear() + self.addCleanup(snm._blocked_port_since.clear) + + def _check(self, cluster, reads): + node = _node("n1") + db = MagicMock() + db.get_primary_storage_nodes_by_secondary_node_id.return_value = [] + db.get_cluster_by_id.return_value = cluster + reads = list(reads) + + def _ports(_n, ports): + own = reads.pop(0) if reads else True + return {p: (own if p == OWN_PORT else True) for p in ports} + sleep = MagicMock() + with patch.object(snm, "db", db), \ + patch.object(snm.health_controller, "check_ports_on_node", side_effect=_ports), \ + patch.object(snm, "_remediate_stale_port_blocks"), \ + patch.object(snm.time, "sleep", sleep): + return snm.node_port_check_fun(node), sleep + + def test_fresh_own_fence_is_not_counted(self): + verdict, sleep = self._check(_cluster(True), [False]) + self.assertTrue(verdict) + sleep.assert_not_called() + + def test_own_fence_past_the_grace_counts(self): + snm._blocked_port_since[("n1", OWN_PORT)] = time.monotonic() - snm.PEER_PORT_FENCE_GRACE_SEC - 1 + verdict, _ = self._check(_cluster(True), [False]) + self.assertFalse(verdict) + + def test_regular_cluster_keeps_the_re_read(self): + verdict, sleep = self._check(_cluster(False), [False, False]) + self.assertFalse(verdict) + sleep.assert_called_once_with(snm.PORT_BLOCK_CONFIRM_SEC) + + +class TestCorrelatedPortFailure(unittest.TestCase): + + def setUp(self): + snm._port_failed_at.clear() + snm._correlated_event_at.clear() + self.addCleanup(snm._port_failed_at.clear) + + def _fail(self, *nids): + for nid in nids: + snm._note_port_check(_node(nid), False) + + def test_within_tolerance_the_node_goes_down(self): + self._fail("n1", "n2") + self.assertEqual(snm._correlated_port_failure(_cluster(True, ftt=2), _node("n2")), "") + + def test_beyond_tolerance_no_node_goes_down(self): + self._fail("n1", "n2", "n3") + with patch.object(snm.storage_events, "snode_port_check_correlated") as ev: + reason = snm._correlated_port_failure(_cluster(True, ftt=2), _node("n3")) + snm._correlated_port_failure(_cluster(True, ftt=2), _node("n1")) + self.assertIn("3 nodes fail only their port check", reason) + ev.assert_called_once() # rate-limited per cluster + + def test_a_passing_check_clears_the_node(self): + self._fail("n1", "n2", "n3") + snm._note_port_check(_node("n1"), True) + self.assertEqual(snm._correlated_port_failure(_cluster(True, ftt=2), _node("n2")), "") + + def test_old_failures_age_out(self): + self._fail("n1", "n2", "n3") + for nid in ("n1", "n2"): + cid, _t = snm._port_failed_at[nid] + snm._port_failed_at[nid] = (cid, time.monotonic() - snm.PORT_FAILURE_CORRELATION_SEC - 1) + self.assertEqual(snm._correlated_port_failure(_cluster(True, ftt=2), _node("n3")), "") + + def test_regular_cluster_never_holds(self): + self._fail("n1", "n2", "n3", "n4") + self.assertEqual(snm._correlated_port_failure(_cluster(False, ftt=1), _node("n1")), "") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_spdk_crash_logs.py b/tests/unit/test_spdk_crash_logs.py new file mode 100644 index 0000000000..54652f8e9f --- /dev/null +++ b/tests/unit/test_spdk_crash_logs.py @@ -0,0 +1,58 @@ +"""The node agent keeps the log tail of an SPDK container that died on its own +before a restart removes it (2026-11-20 FTT2 bed: three SPDKs died in a +cascade and every trace went with their containers).""" + +import json +import os +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_web.api.internal.storage_node import docker as agent + + +def _client(status, logs=b"line1\nline2\n"): + c = MagicMock() + c.attrs = {"State": {"Status": status, "ExitCode": 134, "OOMKilled": False, + "Error": "", "StartedAt": "s", "FinishedAt": "f"}} + c.logs.return_value = logs + client = MagicMock() + client.containers.get.return_value = c + return client + + +class TestPreserveExitedSpdkLogs(unittest.TestCase): + + def setUp(self): + self.dir = tempfile.mkdtemp() + p = patch.object(agent, "SPDK_CRASH_LOG_DIR", self.dir) + p.start() + self.addCleanup(p.stop) + + def test_exited_container_log_is_kept(self): + path = agent._preserve_exited_spdk_logs(_client("exited"), "/spdk_4420") + self.assertTrue(path and os.path.exists(path)) + with open(path, "rb") as f: + head, rest = f.read().split(b"\n", 1) + self.assertEqual(json.loads(head)["ExitCode"], 134) + self.assertIn(b"line2", rest) + + def test_running_container_is_left_alone(self): + self.assertIsNone(agent._preserve_exited_spdk_logs(_client("running"), "/spdk_4420")) + self.assertEqual(os.listdir(self.dir), []) + + def test_only_the_newest_are_kept(self): + with patch.object(agent, "SPDK_CRASH_LOG_KEEP", 2): + for i in range(4): + with patch.object(agent.time, "strftime", return_value=f"t{i}"): + agent._preserve_exited_spdk_logs(_client("exited"), "/spdk_4420") + self.assertEqual(len(os.listdir(self.dir)), 2) + + def test_never_raises(self): + client = MagicMock() + client.containers.get.side_effect = RuntimeError("dockerd wedged") + self.assertIsNone(agent._preserve_exited_spdk_logs(client, "/spdk_4420")) + + +if __name__ == "__main__": + unittest.main() From 0cff315651f3fe88691960cbb00337913429a068 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 20:26:30 +0100 Subject: [PATCH 39/50] feat(array): SCSI LUNs over FC/iSCSI multipath (dm-multipath maps) A SCSI LUN reached over several paths appears once per path as an sd disk and once as the dm-multipath map over them. The Unified DataStore used one raw path, beside multipathd. Now: - node agent inventory reports multipath maps (type mpath, /dev/mapper/, kname dm-N, the identity of their paths, by-id dm-uuid-mpath- first); the path disks are marked with their multipath_map and stay held by it. - find_lun_devices / array_controller._find_in_inventory return the map instead of its paths; a path of a map is never used alone. - resolve_lblk_entries resolves a configured LUN to its map, not a path; the map's dm-N kernel name selects the io_uring engine (9783742ca), which device-mapper multipath needs (it refuses RWF_NOWAIT; data plane: ultra #236). - lio_selftest --multipath: one host over two portals under multipathd: the LUN is found as the map, I/O survives a path loss, growth reaches the map (multipathd resize), a fence fails its I/O, an unfence restores it. 8/8 for all 10 iSCSI vendors (WSL2: multipathd started directly, as systemd skips the unit in a container). Reads use aligned O_DIRECT as SPDK does (dd's O_DIRECT reads from dm fail there). Co-Authored-By: Claude Opus 5.5 --- .../controllers/array_controller.py | 22 +- .../storage_arrays/testing/lio.py | 8 +- .../storage_arrays/testing/lio_selftest.py | 244 +++++++++++++++++- simplyblock_core/utils/__init__.py | 3 +- simplyblock_web/node_utils.py | 81 +++++- tests/unit/test_array_lifecycle.py | 21 ++ tests/unit/test_san_host_agent.py | 76 ++++++ 7 files changed, 427 insertions(+), 28 deletions(-) diff --git a/simplyblock_core/controllers/array_controller.py b/simplyblock_core/controllers/array_controller.py index 01956e0ae6..271c367bf1 100644 --- a/simplyblock_core/controllers/array_controller.py +++ b/simplyblock_core/controllers/array_controller.py @@ -354,14 +354,20 @@ def lun_name(hostname, socket) -> str: def _find_in_inventory(inventory, wwn): + """The host's device of LUN ``wwn``: its multipath map where the LUN is + reached over several paths (FC, iSCSI with several sessions), else its + disk.""" want = multi_attach.normalize_wwn(wwn) - for d in inventory or []: - if d.get("type", "disk") != "disk": - continue - if want in (multi_attach.normalize_wwn(d.get("wwn", "")), - multi_attach.normalize_wwn(d.get("serial", ""))): - return d - return None + matches = [d for d in inventory or [] + if d.get("type", "disk") in ("disk", "mpath") + and want in (multi_attach.normalize_wwn(d.get("wwn", "")), + multi_attach.normalize_wwn(d.get("serial", "")))] + maps = [d for d in matches if d.get("type") == "mpath"] + if maps: + return maps[0] + # A path of a map whose map is not listed (an older agent): never alone. + singles = [d for d in matches if not d.get("multipath_map")] + return singles[0] if singles else None def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configured, @@ -420,6 +426,8 @@ def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configur "numa": int(found.get("numa_node", -1)), "model": found.get("model", ""), "type": "disk", + # A multipath map's kernel name (dm-N) makes it a bdev_uring device. + "kname": found.get("kname", found["name"]), "lun_wwn": lun["wwn"], "array_lun_id": lun["id"], } diff --git a/simplyblock_core/storage_arrays/testing/lio.py b/simplyblock_core/storage_arrays/testing/lio.py index 8ba740b77c..482e7ea7eb 100644 --- a/simplyblock_core/storage_arrays/testing/lio.py +++ b/simplyblock_core/storage_arrays/testing/lio.py @@ -223,7 +223,8 @@ class ConfigfsLio: """The kernel's iSCSI target through rtslib. One target, one portal group, one ACL per initiator; volumes are sparse files on loop devices.""" - def __init__(self, target_iqn: str, data_dir: str, portal: tuple[str, int] = ("0.0.0.0", 3260)): + def __init__(self, target_iqn: str, data_dir: str, + portal: tuple[str, int] | list[tuple[str, int]] = ("0.0.0.0", 3260)): # rtslib keeps its saved configuration here, and refuses to start on a # host where no target ever ran and the directory is missing. os.makedirs("/etc/rtslib-fb-target", exist_ok=True) @@ -240,7 +241,10 @@ def __init__(self, target_iqn: str, data_dir: str, portal: tuple[str, int] = ("0 for name, value in (("authentication", "0"), ("generate_node_acls", "0"), ("demo_mode_write_protect", "0"), ("demo_mode_discovery", "1")): self.tpg.set_attribute(name, value) - rts.NetworkPortal(self.tpg, portal[0], portal[1]) + # Several portals give every host several paths to every LUN: the + # multipath case of FC and iSCSI arrays. + for ip, port in (portal if isinstance(portal, list) else [portal]): + rts.NetworkPortal(self.tpg, ip, port) self.tpg.enable = True def _remove_previous(self, fabric, target_iqn: str) -> None: diff --git a/simplyblock_core/storage_arrays/testing/lio_selftest.py b/simplyblock_core/storage_arrays/testing/lio_selftest.py index d0136c620b..b7730bb3cd 100644 --- a/simplyblock_core/storage_arrays/testing/lio_selftest.py +++ b/simplyblock_core/storage_arrays/testing/lio_selftest.py @@ -120,14 +120,213 @@ def write_pattern(dev: str) -> str: def read_pattern(dev: str) -> str: - r = subprocess.run(["dd", f"if={dev}", "bs=1M", f"count={PATTERN_BYTES >> 20}", "iflag=direct", - "status=none"], capture_output=True) - if r.returncode != 0 or len(r.stdout) != PATTERN_BYTES: - raise OSError(r.stderr.decode(errors="replace").strip() or f"short read of {dev}") - return hashlib.sha256(r.stdout).hexdigest() + """O_DIRECT read into a page-aligned buffer, as SPDK reads the device.""" + import mmap + flags = os.O_RDONLY + if sys.platform == "linux": + flags |= os.O_DIRECT + buf = mmap.mmap(-1, PATTERN_BYTES) + with os.fdopen(os.open(dev, flags), "rb", buffering=0) as f: + n = f.readinto(buf) + if n != PATTERN_BYTES: + raise OSError(f"short read of {dev}: {n} bytes") + return hashlib.sha256(buf[:]).hexdigest() + + +MODULES = ("configfs", "target_core_mod", "iscsi_target_mod", "target_core_iblock", "iscsi_tcp", "loop", + "dm_multipath") + + +MULTIPATH_CONF = """# written by the simplyblock LIO self-test; restored afterwards +defaults { + user_friendly_names yes + find_multipaths yes + polling_interval 1 +} +# LIO's built-in device entry queues for 12 checks; a fenced map fails at once. +overrides { + no_path_retry fail +} +blacklist { + device { + vendor ".*" + product ".*" + } +} +blacklist_exceptions { + device { + vendor "LIO-ORG" + product ".*" + } +} +""" -MODULES = ("configfs", "target_core_mod", "iscsi_target_mod", "target_core_iblock", "iscsi_tcp", "loop") +class Multipathd: + """multipathd with a configuration that claims only the mock's LUNs, for + the duration of the multipath scenario.""" + + CONF = "/etc/multipath.conf" + BACKUP = "/etc/multipath.conf.sb-selftest" + + def __init__(self) -> None: + if os.path.exists(self.CONF): + os.replace(self.CONF, self.BACKUP) + with open(self.CONF, "w") as f: + f.write(MULTIPATH_CONF) + sh("systemctl", "start", "multipathd", check=False) + self.direct = sh("systemctl", "is-active", "multipathd", check=False).stdout.strip() != "active" + if self.direct: + # systemd skips the unit where it detects a container (WSL). + sh("multipathd") + for _ in range(20): + if sh("multipathd", "show", "daemon", check=False).returncode == 0: + break + time.sleep(0.5) + sh("multipathd", "reconfigure", check=False) + + def close(self) -> None: + sh("multipath", "-F", check=False) + if self.direct: + sh("multipathd", "shutdown", check=False) + else: + sh("systemctl", "stop", "multipathd", "multipathd.socket", check=False) + os.unlink(self.CONF) + if os.path.exists(self.BACKUP): + os.replace(self.BACKUP, self.CONF) + + +def map_of(node_utils, wwn: str, paths: int, timeout: float = 30.0) -> dict | None: + """The multipath map of LUN ``wwn`` once it has ``paths`` paths.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + node_utils.scan_for_new_luns() + subprocess.run(["udevadm", "settle"], check=False) + for d in node_utils.find_lun_devices(wwn): + if d.get("type") == "mpath": + live: dict = next((x for x in node_utils.get_block_devices_info() if x["name"] == d["name"]), {}) + if len(live.get("paths", [])) >= paths: + return d + time.sleep(1) + return None + + +def run_multipath(args, setup, vendor: str, portals: list[str], plugin_cls, node_utils, mirror, backend, + server, check) -> None: + """One host reaching every LUN over two portals, under dm-multipath.""" + step("multipath: host A logs in over two portals") + sh("iscsiadm", "-m", "iface", "-I", "sbtest-a", "-o", "new", check=False) + sh("iscsiadm", "-m", "iface", "-I", "sbtest-a", "-o", "update", "-n", "iface.initiatorname", "-v", IQN_A) + for portal in portals: + sh("iscsiadm", "-m", "discovery", "-t", "st", "-p", portal, "-I", "sbtest-a") + sh("iscsiadm", "-m", "node", "-T", TARGET, "--login") + mpd = Multipathd() + lun = None + plugin = None + try: + endpoint = f"https://127.0.0.1:{args.port}" + options = dict(setup.config.options, protocol="iscsi", ca_bundle=open(server.certfile).read()) + plugin = plugin_cls(endpoint=endpoint, options=options, cluster_id="selftest") + plugin.connect(endpoint, {n: setup.credential.get(n) for n in setup.credential.field_names}, options) + host_a = plugin.ensure_host("selftest-a", {"iscsi_iqn": IQN_A}) + + step("M1. create and attach: the LUN is a multipath map with two paths") + lun = plugin.create_lun("selftest-mpath", GIB, {}) + plugin.attach(lun["id"], host_a) + m = map_of(node_utils, lun["wwn"], paths=2) + check(m is not None, f"LUN {lun['wwn']} found as a multipath map ({m and m['device_path']})") + if m is None: + print(sh("systemctl", "is-active", "multipathd", check=False).stdout) + r = sh("multipath", "-v3", check=False) + print("\n".join((r.stdout + r.stderr).splitlines()[-40:])) + print(sh("multipath", "-ll", check=False).stdout) + print(sh("lsblk", "-o", "NAME,KNAME,TYPE,SIZE,SERIAL,WWN,VENDOR", check=False).stdout) + return + dev = m["device_path"] + digest = write_pattern(dev) + try: + check(read_pattern(dev) == digest, "the map reads back what was written over both paths") + except OSError as e: + check(False, f"the map reads back what was written over both paths ({e})") + + step("M2. losing a path does not stop the I/O") + sh("iscsiadm", "-m", "node", "-T", TARGET, "-p", portals[1], "-I", "sbtest-a", "--logout") + errors: list[str] = [] + got = "" + for _ in range(10): + try: + got = read_pattern(dev) + break + except OSError as e: + errors.append(str(e)) + small = subprocess.run(["dd", f"if={dev}", "bs=4k", "count=1", "iflag=direct", + "status=none"], capture_output=True) + errors.append(f"4k read rc={small.returncode} {small.stderr.decode(errors='replace').strip()}") + time.sleep(1) + print(f" reads before success: {len(errors) // 2}, errors: {errors[:4]}") + check(got == digest, "the map reads the data over the remaining path") + if got != digest: + print(sh("multipath", "-ll", check=False).stdout) + print(sh("cat", f"/sys/block/{m['kname']}/queue/max_sectors_kb", + f"/sys/block/{m['kname']}/queue/logical_block_size", check=False).stdout) + for what, cmd in (("path direct", ["dd", "if=/dev/sde", "bs=4k", "count=1", "iflag=direct", + "of=/dev/null"]), + ("map buffered", ["dd", f"if={dev}", "bs=4k", "count=1", "of=/dev/null"]), + ("dm table", ["dmsetup", "table", m["name"]]), + ("dm status", ["dmsetup", "status", m["name"]])): + r = sh(*cmd, check=False) + print(f" {what}: rc={r.returncode} {(r.stdout + r.stderr).strip()[:300]}") + print("\n".join(sh("dmesg", check=False).stdout.splitlines()[-8:])) + return + sh("iscsiadm", "-m", "node", "-T", TARGET, "-p", portals[1], "-I", "sbtest-a", "--login") + check(map_of(node_utils, lun["wwn"], paths=2) is not None, "the path is back in the map") + + step("M3. grow") + plugin.resize(lun["id"], 2 * GIB) + grown = node_utils.rescan_block_device(dev) + check(grown["size"] >= 2 * GIB and any(r.startswith("multipathd:") for r in grown["rescanned"]), + f"the map is {grown['size']} bytes after rescanning its paths and resizing it") + + step("M4. fence: the map's I/O fails") + plugin.fence(lun["id"], host_a) + failed = "" + for _ in range(15): + try: + read_pattern(dev) + except OSError as e: + failed = str(e) + break + time.sleep(1) + check(bool(failed), f"the map's I/O fails after the fence ({failed[:60]})") + + step("M5. unfence: the map serves again") + plugin.unfence(lun["id"], host_a) + sh("iscsiadm", "-m", "session", "--rescan", check=False) + back = "" + for _ in range(30): + sh("multipathd", "reconfigure", check=False) + try: + back = read_pattern(dev) + break + except OSError: + time.sleep(1) + check(back == digest, "the map reads the data again") + + step("M6. delete") + sh("multipath", "-f", m["name"], check=False) + plugin.delete_lun(lun["id"]) + check(mirror.key(lun["id"]) not in backend.list_volumes(), "the LUN is gone from the target") + lun = None + finally: + if plugin is not None and lun is not None: + try: + plugin.delete_lun(lun["id"]) + except Exception as e: # noqa: BLE001 + print(f"cleanup: {e}") + mpd.close() + sh("iscsiadm", "-m", "node", "-T", TARGET, "--logout", check=False) + sh("iscsiadm", "-m", "node", "-T", TARGET, "-o", "delete", check=False) + sh("iscsiadm", "-m", "iface", "-I", "sbtest-a", "-o", "delete", check=False) def prepare_host(portal_ip: str, dummy_portal: bool) -> None: @@ -139,10 +338,12 @@ def prepare_host(portal_ip: str, dummy_portal: bool) -> None: sh("modprobe", module) if sh("mountpoint", "-q", "/sys/kernel/config", check=False).returncode != 0: sh("mount", "-t", "configfs", "none", "/sys/kernel/config") - if dummy_portal and sh("ip", "link", "show", "sbdummy", check=False).returncode != 0: - sh("ip", "link", "add", "sbdummy", "type", "dummy") - sh("ip", "addr", "add", f"{portal_ip}/32", "dev", "sbdummy") - sh("ip", "link", "set", "sbdummy", "up") + if dummy_portal: + if sh("ip", "link", "show", "sbdummy", check=False).returncode != 0: + sh("ip", "link", "add", "sbdummy", "type", "dummy") + sh("ip", "link", "set", "sbdummy", "up") + if f" {portal_ip}/" not in sh("ip", "-o", "addr", "show", "dev", "sbdummy").stdout: + sh("ip", "addr", "add", f"{portal_ip}/32", "dev", "sbdummy") def main() -> int: @@ -153,6 +354,9 @@ def main() -> int: parser.add_argument("--lio-dir", default="/var/tmp/sb-lio-selftest") parser.add_argument("--dummy-portal", action="store_true", help="put the portal address on a dummy interface (WSL)") + parser.add_argument("--multipath", metavar="IP:PORT", + help="run the multipath scenario: a second portal, and one host under dm-multipath " + "(instead of the two-host scenario)") args = parser.parse_args() if sys.platform != "linux": @@ -169,14 +373,18 @@ def main() -> int: setup = module.contract_setup() ip, _, port = args.portal.rpartition(":") prepare_host(ip, args.dummy_portal) - backend = ConfigfsLio(TARGET, args.lio_dir, (ip, int(port))) + portals = [(ip, int(port))] + if args.multipath: + ip2, _, port2 = args.multipath.rpartition(":") + prepare_host(ip2, args.dummy_portal) + portals.append((ip2, int(port2))) + backend = ConfigfsLio(TARGET, args.lio_dir, portals) mirror = LioMirror(setup.fake, setup.config, setup.credential, backend, transport=FakeTransport(setup.fake), allow=[IQN_A, IQN_B]) mirror.reconcile() server = serve(setup.fake, "127.0.0.1", args.port, after_request=mirror.reconcile) threading.Thread(target=server.serve_forever, daemon=True).start() - hosts = Hosts(args.portal) failures: list[str] = [] def check(ok: bool, what: str) -> None: @@ -184,6 +392,18 @@ def check(ok: bool, what: str) -> None: if not ok: failures.append(what) + if args.multipath: + try: + run_multipath(args, setup, vendor, [args.portal, args.multipath], array.plugin_class(vendor), + node_utils, mirror, backend, server, check) + finally: + server.shutdown() + backend_cleanup(backend) + print(f"{'FAILED' if failures else 'PASSED'}: {len(failures)} failure(s)", flush=True) + return 1 if failures else 0 + + hosts = Hosts(args.portal) + plugin = None lun = None try: diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 435fe47829..61e95535de 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -1686,7 +1686,8 @@ def resolve_lblk_entries(configured_entries, host_devices, *, name_fallback=True hard reset of a node fenced from its LUN, the LUN's name nvme2n1 was the node's journal volume, which the restart took up as a data device.""" by_path = _index_by_selector(host_devices) - by_serial = {d["serial"]: d for d in host_devices} + # A path of a multipath map shares the map's serial; the map is the device. + by_serial = {d["serial"]: d for d in host_devices if not d.get("multipath_map")} by_name = {d["name"]: d for d in host_devices} by_partuuid = {d["partuuid"].lower(): d for d in host_devices if d.get("partuuid")} diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index ad0a5a93b9..9d9edd1aac 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -420,10 +420,72 @@ def get_block_devices_info() -> list[dict]: "physical_block_size": physical_bs, "start_bytes": _partition_start_bytes(name, part_name), }) + devices += _multipath_maps(data, devices, persistent) logger.debug("function:get_block_devices_info end") return devices +def _multipath_maps(lsblk: dict, devices: list[dict], persistent: dict) -> list[dict]: + """The dm-multipath maps of the host, one entry each. + + A SCSI LUN reached over several paths (FC, or iSCSI with several + sessions) appears once per path as an sd disk, and once as the map that + multipathd builds over those paths; lsblk lists the map as a child of + every path. The map is what is used: it survives the loss of a path. It + is reported with the identity of its paths (lsblk gives a map neither a + SERIAL nor a WWN), so a LUN is found under the same serial and WWN with + and without multipathing. Its paths stay in the inventory, each marked + with its ``multipath_map`` and held by it, so that nothing takes a path + on its own. The map is opened as /dev/mapper/; its kernel name + (dm-N) is what makes the control plane serve it through bdev_uring.""" + by_name = {d["name"]: d for d in devices if d.get("type") == "disk"} + maps: dict[str, dict] = {} + for dev in lsblk.get("blockdevices", []): + if dev.get("type") != "disk": + continue + for child in dev.get("children") or []: + if child.get("type") != "mpath": + continue + entry = maps.setdefault(child["name"], {"child": child, "paths": []}) + entry["paths"].append(dev["name"]) + out = [] + for map_name, m in sorted(maps.items()): + child, paths = m["child"], m["paths"] + kname = child.get("kname") or map_name + first = by_name.get(paths[0], {}) + for path in paths: + if path in by_name: + by_name[path]["multipath_map"] = map_name + links = persistent.get(kname, []) or persistent.get(map_name, []) + # The stable, host-independent link first: dm-uuid-mpath-. + links = sorted(links, key=lambda link: (0 if "/dm-uuid-mpath-" in link else 1, link)) + out.append({ + "name": map_name, + "kname": kname, + "device_path": f"/dev/mapper/{map_name}", + "type": "mpath", + "size": int(child.get("size") or first.get("size") or 0), + "serial": first.get("serial", ""), + "serial_synthetic": bool(first.get("serial_synthetic")), + "wwn": first.get("wwn", ""), + "model": first.get("model", ""), + "vendor": first.get("vendor", ""), + "rota": bool(first.get("rota")), + "ro": bool(child.get("ro")), + "has_partitions": any(c.get("type") == "part" for c in child.get("children") or []), + "mounted_in_subtree": _subtree_mounted(child), + "holders": _disk_holders(kname), + "is_root_disk": False, + "by_id_path": links[0] if links else "", + "by_id_paths": links, + "numa_node": first.get("numa_node", -1), + "logical_block_size": _logical_block_size(kname), + "physical_block_size": _physical_block_size(kname), + "paths": sorted(paths), + }) + return out + + SB_GPT_PARTITION_TYPECODE = "6527994e-2c5a-4eec-9613-8f5944074e8b" @@ -544,11 +606,14 @@ def find_lun_devices(wwn: str) -> list[dict]: if not want: return [] found = [] - for dev in get_block_devices_info(): - if dev.get("type") != "disk": - continue - if want not in (_normalize_wwn(dev.get("wwn", "")), _normalize_wwn(dev.get("serial", ""))): - continue + devices = get_block_devices_info() + matched = [d for d in devices + if d.get("type") in ("disk", "mpath") + and want in (_normalize_wwn(d.get("wwn", "")), _normalize_wwn(d.get("serial", "")))] + # Several paths of one LUN under a multipath map: the map, not the paths. + if any(d.get("type") == "mpath" for d in matched): + matched = [d for d in matched if d.get("type") == "mpath"] + for dev in matched: found.append({ "name": dev["name"], "device_path": dev["device_path"], @@ -556,7 +621,11 @@ def find_lun_devices(wwn: str) -> list[dict]: "by_id_paths": dev.get("by_id_paths", []), "size": dev.get("size", 0), "ro": dev.get("ro", False), - "has_alceml_signature": simplyblock_data_signature(dev["name"]), + "type": dev.get("type", "disk"), + "kname": dev.get("kname", dev["name"]), + # /dev/ of a map is /dev/mapper/; /dev/ is dm-N. + "has_alceml_signature": simplyblock_data_signature( + dev.get("kname") or dev["name"] if dev.get("type") == "mpath" else dev["name"]), }) return found diff --git a/tests/unit/test_array_lifecycle.py b/tests/unit/test_array_lifecycle.py index b802629cf8..42d0e11497 100644 --- a/tests/unit/test_array_lifecycle.py +++ b/tests/unit/test_array_lifecycle.py @@ -537,6 +537,27 @@ def test_existing_node_initiators_come_from_its_agent(self): self.assertEqual(FakeArrayPlugin.hosts[host]["initiators"]["nvme_hostnqn"], "nqn.h:a") self.assertEqual(n.array_host_id, host) + def test_a_lun_behind_a_multipath_map_is_the_map(self): + cl = _cluster(lun_size=GIB) + api = MagicMock() + + def inv(): + lun = next(iter(FakeArrayPlugin.luns.values()), None) + if lun is None: + return [], None + path = {"type": "disk", "serial": lun["wwn"], "wwn": "", "size": GIB, + "multipath_map": "mpatha", "holders": ["dm-3"]} + return [dict(path, name="sdb", by_id_path="/dev/disk/by-id/wwn-sdb"), + dict(path, name="sdc", by_id_path="/dev/disk/by-id/wwn-sdc"), + {"type": "mpath", "name": "mpatha", "kname": "dm-3", "serial": lun["wwn"], + "wwn": "", "size": GIB, "paths": ["sdb", "sdc"], "numa_node": 0, + "by_id_path": "/dev/disk/by-id/dm-uuid-mpath-3" + lun["wwn"]}], None + api.get_blockdevices.side_effect = inv + api.scan_new_luns.return_value = ({}, None) + entry, _ = array_controller.provision_node_lun(cl, api, {"hostname": "w"}, {"socket": 0}, []) + self.assertEqual((entry["name"], entry["kname"]), ("mpatha", "dm-3")) + self.assertTrue(entry["by_id"].startswith("/dev/disk/by-id/dm-uuid-mpath-")) + def test_lun_that_never_appears_fails(self): cl = _cluster(lun_size=GIB) with self.assertRaisesRegex(array_controller.ArrayLifecycleError, "did not appear"): diff --git a/tests/unit/test_san_host_agent.py b/tests/unit/test_san_host_agent.py index f3371675bc..864a4db50f 100644 --- a/tests/unit/test_san_host_agent.py +++ b/tests/unit/test_san_host_agent.py @@ -94,3 +94,79 @@ def write(p, v): with patch("glob.glob", side_effect=lambda pat: [p for p in paths if "scsi_host" in pat]), \ patch.object(node_utils, "_write_sysfs", side_effect=write): self.assertEqual(node_utils.scan_for_new_luns()["scanned"], paths[1:]) + + +# A SCSI LUN reached over several paths (FC or iSCSI with dm-multipath) is +# one multipath map: the inventory reports the map, with the identity of its +# paths, and a LUN lookup returns the map instead of the paths below it. +_MPATH_LSBLK = {"blockdevices": [ + {"name": "sda", "kname": "sda", "type": "disk", "size": 100 << 30, "serial": "LOCAL1", "wwn": None}, + {"name": "sdb", "kname": "sdb", "type": "disk", "size": 4 << 40, "serial": "624a9370aa", "wwn": "0x624a9370aa", + "children": [{"name": "mpatha", "kname": "dm-3", "type": "mpath", "size": 4 << 40}]}, + {"name": "sdc", "kname": "sdc", "type": "disk", "size": 4 << 40, "serial": "624a9370aa", "wwn": "0x624a9370aa", + "children": [{"name": "mpatha", "kname": "dm-3", "type": "mpath", "size": 4 << 40}]}, + {"name": "sdd", "kname": "sdd", "type": "disk", "size": 2 << 40, "serial": "624a9370bb", "wwn": "0x624a9370bb"}, +]} + + +class TestMultipathInventory(unittest.TestCase): + + def setUp(self): + import json as _json + patches = [ + patch.object(node_utils.shell_utils, "run_command", + return_value=(_json.dumps(_MPATH_LSBLK), "", 0)), + patch.object(node_utils, "_root_disk_names", return_value={"sda"}), + patch.object(node_utils.host_network, "node_hostname", return_value="h1"), + patch.object(node_utils, "read_persistent_links", return_value={ + "dm-3": ["/dev/disk/by-id/dm-uuid-mpath-3624a9370aa", "/dev/disk/by-id/dm-name-mpatha"], + "sdd": ["/dev/disk/by-id/wwn-0x624a9370bb"]}), + patch.object(node_utils, "_read_sysfs", return_value=""), + patch.object(node_utils, "_logical_block_size", return_value=512), + patch.object(node_utils, "_physical_block_size", return_value=4096), + patch.object(node_utils, "_disk_holders", + side_effect=lambda n: ["dm-3"] if n in ("sdb", "sdc") else []), + patch.object(node_utils, "_subtree_mounted", return_value=False), + patch.object(node_utils, "simplyblock_data_signature", return_value=False), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + + def test_the_map_is_reported_with_its_paths_identity(self): + devices = {d["name"]: d for d in node_utils.get_block_devices_info()} + m = devices["mpatha"] + self.assertEqual((m["type"], m["kname"], m["device_path"]), ("mpath", "dm-3", "/dev/mapper/mpatha")) + self.assertEqual((m["serial"], m["wwn"], m["size"]), ("624a9370aa", "0x624a9370aa", 4 << 40)) + self.assertEqual(m["paths"], ["sdb", "sdc"]) + self.assertEqual(m["by_id_path"], "/dev/disk/by-id/dm-uuid-mpath-3624a9370aa") + self.assertEqual(m["holders"], []) + # The paths stay listed, held by the map, so nothing takes them alone. + self.assertEqual(devices["sdb"]["multipath_map"], "mpatha") + self.assertEqual(devices["sdb"]["holders"], ["dm-3"]) + + def test_a_lun_lookup_returns_the_map_not_its_paths(self): + found = node_utils.find_lun_devices("naa.624a9370aa") + self.assertEqual([d["name"] for d in found], ["mpatha"]) + self.assertEqual(found[0]["device_path"], "/dev/mapper/mpatha") + self.assertEqual(found[0]["by_id_path"], "/dev/disk/by-id/dm-uuid-mpath-3624a9370aa") + + def test_a_single_path_lun_is_still_its_disk(self): + found = node_utils.find_lun_devices("624a9370bb") + self.assertEqual([d["name"] for d in found], ["sdd"]) + + +class TestMultipathResolution(unittest.TestCase): + + def test_a_configured_lun_resolves_to_its_map_not_a_path(self): + from simplyblock_core import utils + path = {"type": "disk", "serial": "624a9370aa", "size": 1, "holders": ["dm-3"], + "multipath_map": "mpatha"} + live = [dict(path, name="sdb", device_path="/dev/sdb"), + {"type": "mpath", "name": "mpatha", "kname": "dm-3", "serial": "624a9370aa", "size": 1, + "device_path": "/dev/mapper/mpatha", "holders": []}, + dict(path, name="sdc", device_path="/dev/sdc")] + resolved, missing = utils.resolve_lblk_entries([{"name": "mpatha", "serial": "624a9370aa"}], live) + self.assertEqual(missing, []) + self.assertEqual(resolved[0]["current_path"], "/dev/mapper/mpatha") + self.assertEqual(resolved[0]["io_engine"], "uring") From 94656c15d535c39e7e635bc080a0fbdac7fa0ab0 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 20:42:29 +0100 Subject: [PATCH 40/50] multi-attach: warm standby alcemls for fast LUN takeover (opt-in) Problem: a LUN takeover opens the LUN's alceml cold on the standby (pba_init_mode=1), which loads the whole page map. That is fine on 20 GiB but takes minutes at 100 TiB, far beyond the peers' I/O hold. Design: the ultra data plane (PR #236) supports standby alceml instances (standby=true, open-only) that keep their page map warm from the LUN's region counters (refresh_interval_ms) and activate on their first I/O: fence by reservation preemption, epoch bump, final refresh, then serve. - Opt-in per cluster: Cluster.multi_attach_warm_standby (immutable), --multi-attach-warm-standby on cluster create/add, v1/v2 APIs; needs multi_attach. multi_attach.warm_standby_enabled(cluster, dev) also requires the device's reservations (a standby fences by preemption). - lun_takeover.ensure_warm_standbys(node), driven by the storage-node monitor (own thread per node, at most every 30 s, ONLINE nodes of warm clusters only): opens a standby alceml (open-only, the node's reservation key, the takeover's parameters) on every attached, non-serving, unfenced node of an ONLINE LUN, over the node's own path (agent lun_info + pick_lun_path). Never exported. Closes the standby (and, on a non-owner, its base bdev) when the node is no longer a standby; never an exported stack, an active instance or the serving node's. retire_stale_local_stack keeps a kept warm standby. - Takeover: when the chosen standby holds a standby instance (alceml_get_role), it is reused: no alceml create and no activation wait before the export; PT/subsystem/listeners/ns are built, the record and paths switched as before, then the activation (driven by the peers' retried I/O) is awaited. If it does not activate: array fence + fenced record, export torn down, LUN reopened cold on the same standby (same NQN/IPs). If that fails too the device is set unavailable. A standby whose activation failed (act_failed) is closed and reopened active. Without a warm standby the cold path is unchanged. - Memory: the page map counts in the node's SPDK system-memory budget (sys_memory_for_capacity, 0.7 GiB per raw TiB). A restart now sizes it over the node's own devices plus its warm-standby LUNs. At add-node the standby set is not known yet (documented); ensure_warm_standbys does not open a standby beyond the node's budget and raises a cluster event instead. - rpc_client.bdev_alceml_create: refresh_interval_ms (standby only). Co-Authored-By: Claude Opus 5.5 --- simplyblock_cli/cli-reference.yaml | 16 + simplyblock_cli/cli.py | 2 + simplyblock_cli/clibase.py | 2 + simplyblock_core/cluster_ops.py | 15 + simplyblock_core/constants.py | 7 + simplyblock_core/controllers/lun_takeover.py | 353 ++++++++++++- simplyblock_core/models/cluster.py | 6 + simplyblock_core/multi_attach.py | 50 ++ simplyblock_core/rpc_client.py | 6 +- .../services/storage_node_monitor.py | 36 ++ simplyblock_core/storage_node_ops.py | 27 + simplyblock_web/api/v1/cluster.py | 1 + simplyblock_web/api/v2/_dtos.py | 2 + simplyblock_web/api/v2/cluster/__init__.py | 3 + tests/unit/test_lun_warm_standby.py | 476 ++++++++++++++++++ 15 files changed, 985 insertions(+), 17 deletions(-) create mode 100644 tests/unit/test_lun_warm_standby.py diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 9c23422597..31ec721bbe 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1056,6 +1056,14 @@ commands: dest: multi_attach type: bool action: store_true + - name: "--multi-attach-warm-standby" + help: > + Multi-attach only: keep a warm standby instance of every LUN on each node it is attached to but not + served by. A takeover then activates it on the first I/O instead of loading the LUN's page map cold + (minutes on large LUNs). Each standby holds the LUN's page map in hugepages on its node. + dest: multi_attach_warm_standby + type: bool + action: store_true - name: "--ftt" help: > Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least @@ -1371,6 +1379,14 @@ commands: dest: multi_attach type: bool action: store_true + - name: "--multi-attach-warm-standby" + help: > + Multi-attach only: keep a warm standby instance of every LUN on each node it is attached to but not + served by. A takeover then activates it on the first I/O instead of loading the LUN's page map cold + (minutes on large LUNs). Each standby holds the LUN's page map in hugepages on its node. + dest: multi_attach_warm_standby + type: bool + action: store_true - name: "--ftt" help: > Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 5f3d845cb8..61259a2cd8 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -470,6 +470,7 @@ def init_cluster__create(self, subparser): subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ nodes); the flag only affects status reporting.', dest='strict_node_anti_affinity', action='store_true') subcommand.add_argument('--raid-devices', help='Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no other device is writable. Cannot be changed after cluster creation.', dest='raid_devices', action='store_true') subcommand.add_argument('--multi-attach', help='Multi-attach array LUN mode. Every data device is a LUN of an external array that already protects it and presents it to several storage nodes. The cluster runs 1+0 erasure coding (the data-chunks and parity-chunks settings are overridden); on a node failure the LUN is reopened on a standby node and reached over an additional NVMe-oF path. Requires --raid-devices, --ftt and --ha-type ha. Cannot be changed after cluster creation.', dest='multi_attach', action='store_true') + subcommand.add_argument('--multi-attach-warm-standby', help='Multi-attach only: keep a warm standby instance of every LUN on each node it is attached to but not served by. A takeover then activates it on the first I/O instead of loading the LUN\'s page map cold (minutes on large LUNs). Each standby holds the LUN\'s page map in hugepages on its node.', dest='multi_attach_warm_standby', action='store_true') subcommand.add_argument('--ftt', help='Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least 3 storage nodes). Required with --multi-attach, refused without it.', type=int, dest='ftt', choices=[1,2,]) subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') subcommand.add_argument('--disable-lvol-geometry', help='Create new logical volumes with ndcs=npcs=0 instead of the cluster\'s erasure coding geometry. Enabled by default; can be changed later with `cluster set-lvol-geometry`.', dest='disable_lvol_geometry', action='store_true') @@ -523,6 +524,7 @@ def init_cluster__add(self, subparser): subcommand.add_argument('--strict-node-anti-affinity', help='Enable strict node anti affinity for storage nodes. Never more than one chunk is placed on a node. The cluster needs the same node floor as every cluster (_data-chunks-per-stripe + 2 x parity-chunks-per-stripe_ nodes); the flag only affects status reporting.', dest='strict_node_anti_affinity', action='store_true') subcommand.add_argument('--raid-devices', help='Declare that every data device is protected below simplyblock (hardware RAID LUN, md or dm RAID). Lowers the data-device floor from n+2k to n+k and allows a stripe to place several chunks on one device when no other device is writable. Cannot be changed after cluster creation.', dest='raid_devices', action='store_true') subcommand.add_argument('--multi-attach', help='Multi-attach array LUN mode. Every data device is a LUN of an external array that already protects it and presents it to several storage nodes. The cluster runs 1+0 erasure coding (the data-chunks and parity-chunks settings are overridden); on a node failure the LUN is reopened on a standby node and reached over an additional NVMe-oF path. Requires --raid-devices, --ftt and --ha-type ha. Cannot be changed after cluster creation.', dest='multi_attach', action='store_true') + subcommand.add_argument('--multi-attach-warm-standby', help='Multi-attach only: keep a warm standby instance of every LUN on each node it is attached to but not served by. A takeover then activates it on the first I/O instead of loading the LUN\'s page map cold (minutes on large LUNs). Each standby holds the LUN\'s page map in hugepages on its node.', dest='multi_attach_warm_standby', action='store_true') subcommand.add_argument('--ftt', help='Node failures to tolerate in multi-attach mode (1 or 2; 2 adds a tertiary LVS node and needs at least 3 storage nodes). Required with --multi-attach, refused without it.', type=int, dest='ftt', choices=[1,2,]) subcommand.add_argument('--enable-failure-domain', help='Enable failure-domain anti-affinity. Each storage node must then be added with a --failure-domain tag (rack/cabinet/DC); data, journal and secondary/tertiary copies are spread across distinct failure domains (best-effort). Deploy-time only: a cluster cannot be upgraded into this feature, it must be redeployed.', dest='enable_failure_domain', action='store_true') subcommand.add_argument('--disable-lvol-geometry', help='Create new logical volumes with ndcs=npcs=0 instead of the cluster\'s erasure coding geometry. Enabled by default; can be changed later with `cluster set-lvol-geometry`.', dest='disable_lvol_geometry', action='store_true') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 9733aee3f7..408c1c9a30 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -2055,6 +2055,7 @@ def cluster_add(self, args): raid_devices=raid_devices, multi_attach=multi_attach, ftt=ftt, + multi_attach_warm_standby=getattr(args, 'multi_attach_warm_standby', False), container_image_prefix=args.container_image_prefix, ) @@ -2141,6 +2142,7 @@ def cluster_create(self, args): raid_devices=raid_devices, multi_attach=multi_attach, ftt=ftt, + multi_attach_warm_standby=getattr(args, 'multi_attach_warm_standby', False), container_image_prefix=args.container_image_prefix, ) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 8413e2bdc5..29c5a2dded 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -467,6 +467,13 @@ def _resolve_multi_attach(multi_attach, ftt, raid_devices, ha_type, is_single_no return MULTI_ATTACH_SCHEME[0], MULTI_ATTACH_SCHEME[1], int(ftt) +def _validate_warm_standby(multi_attach, warm_standby): + """Warm standbys (Cluster.multi_attach_warm_standby) are a multi-attach + option only.""" + if warm_standby and not multi_attach: + raise ValueError("multi_attach_warm_standby requires multi_attach") + + def _validate_integrity_flags(inline_checksum, atomic_4k): """atomic_4k only lifts the data plane's 4K gate on fallback-mode checksums, so without inline_checksum it is a setting that does nothing.""" @@ -497,10 +504,12 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, raid_devices=False, multi_attach=False, ftt=None, + multi_attach_warm_standby=False, ) -> str: distr_ndcs, distr_npcs, max_fault_tolerance = _resolve_multi_attach( multi_attach, ftt, raid_devices, ha_type, is_single_node, distr_ndcs, distr_npcs, max_fault_tolerance) + _validate_warm_standby(multi_attach, multi_attach_warm_standby) if (distr_ndcs, distr_npcs) not in SUPPORTED_ERASURE_CODING_SCHEMES: raise ValueError("Unsupported erasure coding scheme") _validate_integrity_flags(inline_checksum, atomic_4k) @@ -673,6 +682,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.md_journal = bool(md_journal) cluster.raid_devices = bool(raid_devices) cluster.multi_attach = bool(multi_attach) + cluster.multi_attach_warm_standby = bool(multi_attach and multi_attach_warm_standby) cluster.client_data_nic = client_data_nic or "" cluster.max_fault_tolerance = max_fault_tolerance cluster.nvmf_base_port = nvmf_base_port @@ -830,6 +840,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn raid_devices=False, multi_attach=False, ftt=None, + multi_attach_warm_standby=False, snode_api_service=constants.K8S_SNODE_API_SERVICE, spdk_proxy_service=constants.K8S_SPDK_PROXY_SERVICE, spdk_proxy_tls_secret=constants.K8S_SPDK_PROXY_TLS_SECRET, @@ -872,6 +883,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn raid_devices=raid_devices, multi_attach=multi_attach, ftt=ftt, + multi_attach_warm_standby=multi_attach_warm_standby, snode_api_service=snode_api_service, spdk_proxy_service=spdk_proxy_service, spdk_proxy_tls_secret=spdk_proxy_tls_secret, @@ -914,6 +926,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca raid_devices=False, multi_attach=False, ftt=None, + multi_attach_warm_standby=False, snode_api_service=constants.K8S_SNODE_API_SERVICE, spdk_proxy_service=constants.K8S_SPDK_PROXY_SERVICE, spdk_proxy_tls_secret=constants.K8S_SPDK_PROXY_TLS_SECRET, @@ -931,6 +944,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca distr_ndcs, distr_npcs, max_fault_tolerance = _resolve_multi_attach( multi_attach, ftt, raid_devices, ha_type, is_single_node, distr_ndcs, distr_npcs, max_fault_tolerance) + _validate_warm_standby(multi_attach, multi_attach_warm_standby) if (distr_ndcs, distr_npcs) not in SUPPORTED_ERASURE_CODING_SCHEMES: raise ValueError("Unsupported erasure coding scheme") @@ -1071,6 +1085,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca cluster.md_journal = bool(md_journal) cluster.raid_devices = bool(raid_devices) cluster.multi_attach = bool(multi_attach) + cluster.multi_attach_warm_standby = bool(multi_attach and multi_attach_warm_standby) cluster.client_data_nic = client_data_nic or "" cluster.max_fault_tolerance = max_fault_tolerance cluster.nvmf_base_port = nvmf_base_port diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index 0ebbe7ad52..e804e84ce6 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -1363,6 +1363,13 @@ def get_config_var(name, default=None): MULTI_ATTACH_RESERVATIONS = True # A takeover waits this long for its alceml to report the reservation taken. MULTI_ATTACH_ACTIVATION_WAIT_SEC = 5 +# Warm standby (Cluster.multi_attach_warm_standby): ms between two refreshes +# of a standby alceml's page map from the LUN's region counters (ultra #236, +# refresh_interval_ms; the data plane's default is the same). +MULTI_ATTACH_WARM_REFRESH_MS = 2000 +# How often the storage-node monitor reconciles a node's warm standbys +# (lun_takeover.ensure_warm_standbys), per node. +MULTI_ATTACH_WARM_STANDBY_INTERVAL_SEC = 30 # Client keep-alive on multi-attach clusters: a node lost without closing its # connections (hard reset, network cut) leaves its own volumes' paths dead # until the host's keep-alive notices, 1-2 intervals: 8 s took 10-14 s on the diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index c676bce728..7afabe48bf 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -36,6 +36,18 @@ open-only. When no standby can read the LUN, nothing is changed: peers keep holding until their hold expires, and the caller marks the device unavailable. + +Warm standby (opt-in, ``Cluster.multi_attach_warm_standby``): every attached +non-serving node keeps a standby alceml of the LUN open (``standby=true``, +never exported) whose page map the data plane keeps warm +(:func:`ensure_warm_standbys`, driven by the storage-node monitor). Step 4 +then reuses that instance instead of opening the LUN cold (a full page-map +load, minutes at 100 TiB): it only adds the PT bdev, subsystem, listeners and +namespace. A standby activates on its first I/O (fence by preemption, epoch +bump, final refresh), so the takeover waits for the activation after step 6, +when the peers' held I/O reaches it. If it does not activate, the failed node +is fenced at the array, the export on the standby is torn down and redone +cold (the same NQN and IPs: the paths already point there). """ from __future__ import annotations @@ -307,6 +319,9 @@ def _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) # 6 + 7 (design steps 5 and 7). New path on every ONLINE node, old one off. added = _switch_paths(db, cluster_id, dev, standby, failed, _failed_paths(failed, old_ips, old_port)) + if export.get("warm") and not _activate_warm_standby( + db, cluster, owner, dev, standby, failed, path, export, deadline): + return FAILED if dev.status != NVMeDevice.STATUS_ONLINE: # The hold had expired and the device was marked unavailable; it is @@ -517,7 +532,61 @@ def _create_base_bdev(rpc, name, path, dev): return True -def _export_on_standby(db, cluster, standby, dev, path, deadline) -> dict: +def _alceml_open_kwargs(cluster, dev, extra) -> dict: + """``create_alceml`` kwargs of an open-only alceml of ``dev`` (takeover, + warm standby): never formats, the cluster's page and checksum settings, + plus ``extra`` (multi_attach.alceml_create_kwargs).""" + checksum_method, cache_size, cache_eviction = utils.alceml_checksum_params(cluster, dev) + return dict( + pba_init_mode=ALCEML_OPEN_ONLY, + write_protection=cluster.distr_ndcs > 1, + pba_page_size=cluster.page_size_in_blocks, + full_page_unmap=cluster.full_page_unmap, + checksum_method=checksum_method, + cache_size=cache_size, + cache_eviction_threshold=cache_eviction, + force_4k_atomic=getattr(cluster, 'atomic_4k', False), + **extra, + ) + + +def _alceml_role(node, alceml) -> dict: + """``alceml_get_role`` of ``alceml`` on ``node`` (fast, no retries); {} + when it does not answer.""" + try: + return node.rpc_client(timeout=1, retry=0).alceml_get_role(alceml) or {} + except Exception as e: # noqa: BLE001 + logger.info("alceml_get_role of %s on %s failed: %s", alceml, node.get_id(), e) + return {} + + +def _is_standby_instance(role) -> bool: + """A standby alceml instance that can still activate on its first I/O + (opened standby, or an active instance that was deposed).""" + return role.get("role") == "standby" and role.get("state") != "act_failed" + + +def _reuse_warm_standby(rpc, standby, alceml): + """The standby node holds ``alceml`` already: ``(warm, exists)``. A + standby instance is reused as is (warm). One whose activation failed is + closed, so that it is opened again active; anything else (an active + instance) is left to the regular path.""" + role = _alceml_role(standby, alceml) + if _is_standby_instance(role): + logger.info("LUN takeover: reusing the warm standby %s on %s (warm %s, %s refreshes)", + alceml, standby.get_id(), role.get("warm"), role.get("refreshes_count")) + return True, True + if role.get("role") == "standby": + logger.warning("LUN takeover: standby %s on %s failed its activation (%s); reopening it", + alceml, standby.get_id(), role) + _teardown(rpc, standby.get_id(), [("alceml", alceml)]) + if rpc.bdev_get(alceml): + raise TakeoverError(f"failed standby alceml {alceml} not closed on {standby.get_id()}") + return False, False + return False, True + + +def _export_on_standby(db, cluster, standby, dev, path, deadline, allow_warm=True) -> dict: if getattr(dev, "bdev_type", "aio") != "aio": raise TakeoverError(f"device type {dev.bdev_type!r} cannot be taken over (array LUNs are lblk)") rpc = standby.rpc_client(timeout=_remaining(deadline, 60)) @@ -555,24 +624,18 @@ def _export_on_standby(db, cluster, standby, dev, path, deadline) -> dict: alceml = dev.alceml_bdev or f"alceml_{dev.get_id()}" kwargs = multi_attach.alceml_create_kwargs(dev, standby, cluster) - if not rpc.bdev_get(alceml): - checksum_method, cache_size, cache_eviction = utils.alceml_checksum_params(cluster, dev) - ok = standby.create_alceml( - alceml, base, dev.get_id(), - pba_init_mode=ALCEML_OPEN_ONLY, - write_protection=cluster.distr_ndcs > 1, - pba_page_size=cluster.page_size_in_blocks, - full_page_unmap=cluster.full_page_unmap, - checksum_method=checksum_method, - cache_size=cache_size, - cache_eviction_threshold=cache_eviction, - force_4k_atomic=getattr(cluster, 'atomic_4k', False), - **kwargs, - ) + exists = rpc.bdev_get(alceml) + warm = False + if (exists and allow_warm and kwargs.get("reservation_key") + and multi_attach.warm_standby_enabled(cluster, dev)): + warm, exists = _reuse_warm_standby(rpc, standby, alceml) + if not exists: + ok = standby.create_alceml(alceml, base, dev.get_id(), + **_alceml_open_kwargs(cluster, dev, kwargs)) if not ok: raise TakeoverError(f"alceml {alceml} not opened on {sid}") created.append(("alceml", alceml)) - if kwargs.get("reservation_key"): + if kwargs.get("reservation_key") and not warm: # Opened active with a key: it preempts the old holder (fence). _wait_active(standby.rpc_client(timeout=1, retry=0), alceml, deadline) try: @@ -629,6 +692,10 @@ def _export_on_standby(db, cluster, standby, dev, path, deadline) -> dict: "port": standby.nvmf_port, "base": base, "pt_uuid": aliases[0] if aliases else "", + # A reused warm standby: activates on the first I/O, after the paths. + "warm": warm, + "alceml": alceml, + "pt": pt, } @@ -651,6 +718,54 @@ def _teardown(rpc, sid, created): logger.warning("LUN takeover cleanup on %s: %s %s not removed: %s", sid, kind, name, e) +def _activate_warm_standby(db, cluster, owner, dev, standby, failed, path, export, deadline) -> bool: + """After the paths were switched to a reused warm standby: wait until the + peers' I/O has activated it. If it does not, fence the failed node at the + array, tear the export down and redo it cold (alceml opened active with + the key, as without warm standby); the NQN and IPs stay, so the peers' + paths already point there. Returns False (device set unavailable, event + raised) only when the cold export failed as well.""" + sid, device_id, alceml = standby.get_id(), dev.get_id(), export["alceml"] + try: + _wait_active(standby.rpc_client(timeout=1, retry=0), alceml, deadline) + try: + from simplyblock_core.controllers import qos_controller + standby.rpc_client(timeout=_remaining(deadline, 10)).alceml_set_qos_weights( + qos_controller.get_qos_weights_list(cluster.get_id())) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: QoS weights not set on %s: %s", sid, e) + return True + except TakeoverError as e: + msg = (f"LUN takeover of device {device_id}: warm standby {alceml} on {sid} did not " + f"activate ({e}); fencing node {failed.get_id()} at the array and opening the LUN cold") + logger.warning(msg) + _event(owner.cluster_id, dev, msg, EventObj.LEVEL_WARN) + try: + _fence(cluster, dev, failed) + _record_fenced(db, owner, device_id, failed.get_id()) + rpc = standby.rpc_client(timeout=_remaining(deadline, 60)) + _teardown(rpc, sid, [("alceml", alceml), ("pt", export["pt"]), ("subsystem", export["nqn"])]) + if rpc.bdev_get(alceml): + raise TakeoverError(f"standby alceml {alceml} not closed on {sid}") + _export_on_standby(db, cluster, standby, dev, path, deadline, allow_warm=False) + logger.info("LUN takeover of %s: opened cold on %s after the warm standby failed", + device_id, sid) + return True + except Exception as e: # noqa: BLE001 + msg = (f"LUN takeover of device {device_id} from node {failed.get_id()} failed: the warm " + f"standby on {sid} did not activate and the cold open failed: {e}") + logger.error(msg) + _event(owner.cluster_id, dev, msg, EventObj.LEVEL_CRITICAL) + # The record names the standby, so the takeover worker would not mark + # the device lost: do it here (the regular node-loss behaviour). + try: + from simplyblock_core.controllers import device_controller + device_controller.device_set_unavailable(device_id) + except Exception as e2: # noqa: BLE001 + logger.error("LUN takeover of %s: device not set unavailable: %s", device_id, e2) + return False + + # --------------------------------------------------------------- 5. record def _record_serving(db, owner, device_id, failed_node_id, standby, path, export) -> bool: @@ -785,6 +900,8 @@ def retire_stale_local_stack(node) -> list: has_stack = False if not has_stack: continue + if _is_kept_warm_standby(db, rpc, node, dev, pt): + continue name = f"remote_{dev.alceml_bdev}" own_ips = [iface.ip4_address for iface in node.data_nics or [] if iface.ip4_address] for ip in own_ips: @@ -803,6 +920,23 @@ def retire_stale_local_stack(node) -> list: return retired +def _is_kept_warm_standby(db, rpc, node, dev, pt) -> bool: + """Whether what ``node`` holds of ``dev`` is only the warm standby + ensure_warm_standbys keeps there (an unexported instance opened standby) + on a node not fenced from the LUN: not a stale stack. Always False + without warm standby.""" + if multi_attach.fenced_from(dev, node.get_id()): + return False + try: + cluster = db.get_cluster_by_id(node.cluster_id) + if not multi_attach.warm_standby_enabled(cluster, dev) or rpc.bdev_get(pt): + return False + except Exception: # noqa: BLE001 + return False + role = _alceml_role(node, dev.alceml_bdev) + return _is_standby_instance(role) and role.get("standby_at_open") is True + + def reintegrate_fenced_node(node) -> list: """Make a node that is back ONLINE a standby again for every LUN a takeover fenced it from: unfence it at the array (re-present the LUN). @@ -936,3 +1070,190 @@ def _mut(n): logger.warning("Array unfence of %s on LUN %s failed (%s); the attachment " "reconcile presents it", node_id, dev.lun_wwn, e) return True + + +# ---------------------------------------------------------- warm standby + +# node id -> the device ids whose warm standby did not fit its memory budget +# when last reported (an event is raised when the set changes). +_warm_over_budget: dict[str, frozenset] = {} + + +def ensure_warm_standbys(node) -> dict: + """Keep the warm standby alcemls of an ONLINE node in line with the + record (``Cluster.multi_attach_warm_standby`` clusters only). + + For every LUN the node is a standby of (multi_attach.warm_standby_devices: + attached, not serving, not fenced, the device ONLINE), open a standby + alceml (``standby=true``, open-only, the reservation key, the same + parameters as a takeover) over the node's own path to the LUN, unless one + is open already. It is never exported: a peer's I/O would activate it. + + Close the standby alceml (and, on a node that does not own the device, + its base bdev) of every LUN the node is no longer a standby of (fenced, + detached, removed). Never touches the serving node's stack, an exported + alceml (a PT bdev over it: retire_stale_local_stack handles those) or an + active instance, nor a device whose takeover runs in this process. + + A standby holds the LUN's page map in the node's SPDK memory: one is not + opened when the node's budget (StorageNode.minimum_sys_memory, its + container limit) does not cover its own devices plus its standbys + (utils.sys_memory_for_capacity); a cluster event says so instead. A + restart sizes the budget for them (storage_node_ops). + + Returns ``{"created": [...], "removed": [...], "over_budget": [...]}`` + (device ids); best-effort and idempotent, never raises.""" + report: dict = {"created": [], "removed": [], "over_budget": []} + try: + _ensure_warm_standbys(node, report) + except Exception as e: # noqa: BLE001 + logger.error("Warm standbys of %s not reconciled: %s", node.get_id(), e) + return report + + +def _ensure_warm_standbys(node, report): + node_id = node.get_id() + db = DBController() + cluster = db.get_cluster_by_id(node.cluster_id) + if not (multi_attach.is_multi_attach(cluster) + and getattr(cluster, "multi_attach_warm_standby", False) is True): + return + nodes = db.get_storage_nodes_by_cluster_id(node.cluster_id) + node = next((n for n in nodes if n.get_id() == node_id), node) + if node.status != StorageNode.STATUS_ONLINE: + return + keep = {dev.get_id() for _, dev in multi_attach.warm_standby_devices( + cluster, node_id, nodes, multi_attach._WARM_STANDBY_BUDGET_STATES)} + want = multi_attach.warm_standby_devices(cluster, node_id, nodes) + try: + rpc = node.rpc_client(timeout=10, retry=1) + except Exception as e: # noqa: BLE001 + logger.info("Warm standbys: no RPC client for %s: %s", node_id, e) + return + + # Close what the node is no longer a standby of. + for owner in nodes: + for dev in owner.nvme_devices or []: + if (not multi_attach.is_multi_attach(dev) or dev.get_id() in keep + or dev.serving_node() == node_id or _local_lock(dev.get_id()).locked()): + continue + if _remove_warm_standby(rpc, node, dev): + report["removed"].append(dev.get_id()) + + # Open what is missing, within the node's memory budget. + committed = sum(max(int(d.size or 0), 0) for d in node.nvme_devices or []) + committed += sum(max(int(d.size or 0), 0) for o, d in multi_attach.devices_served_by(nodes, node_id) + if o.get_id() != node_id) + missing = [] + for _owner, dev in want: + try: + held = bool(rpc.bdev_get(dev.alceml_bdev)) + except Exception as e: # noqa: BLE001 + logger.info("Warm standbys: SPDK of %s not answering: %s", node_id, e) + return + if held: + committed += max(int(dev.size or 0), 0) + else: + missing.append(dev) + budget = int(getattr(node, "minimum_sys_memory", 0) or 0) + for dev in missing: + if _local_lock(dev.get_id()).locked(): + continue + size = max(int(dev.size or 0), 0) + if budget and utils.sys_memory_for_capacity(committed + size) > budget: + report["over_budget"].append(dev.get_id()) + continue + if _create_warm_standby(rpc, cluster, node, dev): + report["created"].append(dev.get_id()) + committed += size + _report_over_budget(cluster, node, report["over_budget"], committed, budget) + + +def _create_warm_standby(rpc, cluster, node, dev) -> bool: + """Open the standby alceml of ``dev`` on ``node``; False (logged) when the + node cannot read the LUN or SPDK refuses it. Closes what it opened on + failure.""" + node_id = node.get_id() + kwargs = multi_attach.alceml_create_kwargs(dev, node, cluster) + if not kwargs.get("reservation_key") or getattr(dev, "bdev_type", "aio") != "aio": + return False + path, why = _probe(node, dev, time.monotonic() + 15) + if not path: + logger.info("Warm standby of %s on %s not opened: %s", dev.get_id(), node_id, why) + return False + base = utils.aio_bdev_name_for_serial(dev.serial_number) + created = [] + try: + if not rpc.bdev_get(base): + _create_base_bdev(rpc, base, path, dev) + created.append(("base", base)) + info = rpc.bdev_get(base) or {} + size = int(info.get("block_size", 0)) * int(info.get("num_blocks", 0)) + if size < int(dev.size or 0): + raise TakeoverError(f"base bdev {base} is {size} bytes, the device {dev.size}") + ok = node.create_alceml(dev.alceml_bdev, base, dev.get_id(), + standby=True, + refresh_interval_ms=constants.MULTI_ATTACH_WARM_REFRESH_MS, + **_alceml_open_kwargs(cluster, dev, kwargs)) + if not ok: + raise TakeoverError(f"standby alceml {dev.alceml_bdev} not opened") + created.append(("alceml", dev.alceml_bdev)) + except Exception as e: # noqa: BLE001 + logger.warning("Warm standby of %s on %s not opened: %s", dev.get_id(), node_id, e) + _teardown(rpc, node_id, created) + return False + logger.info("Warm standby of %s (LUN %s) opened on %s at %s", + dev.get_id(), dev.lun_wwn, node_id, path) + return True + + +def _remove_warm_standby(rpc, node, dev) -> bool: + """Close a standby alceml of ``dev`` the node holds although it is no + longer a standby of the LUN; its base bdev too unless the node owns the + device. Only an unexported standby instance is closed.""" + alceml = dev.alceml_bdev + try: + if not alceml or not rpc.bdev_get(alceml): + return False + if rpc.bdev_get(dev.pt_bdev or f"{alceml}_PT"): + return False + except Exception: # noqa: BLE001 + return False + if _alceml_role(node, alceml).get("role") != "standby": + return False + todo = [("alceml", alceml)] + if dev.node_id != node.get_id(): + todo.insert(0, ("base", utils.aio_bdev_name_for_serial(dev.serial_number))) + _teardown(rpc, node.get_id(), todo) + logger.info("Warm standby of %s closed on %s: no longer a standby of LUN %s", + dev.get_id(), node.get_id(), dev.lun_wwn) + return True + + +def _report_over_budget(cluster, node, device_ids, committed, budget): + """A cluster event (once per change) when warm standbys of the node did + not fit its memory budget.""" + node_id = node.get_id() + now = frozenset(device_ids) + if _warm_over_budget.get(node_id, frozenset()) == now: + return + _warm_over_budget[node_id] = now + if not now: + return + msg = (f"Node {node_id}: warm standbys of {len(now)} LUN(s) ({', '.join(sorted(now))}) not " + f"opened: its SPDK memory budget {utils.humanbytes(budget)} covers its own devices and " + f"standbys of {utils.humanbytes(committed)} only. Restart the node to size its memory " + f"for its standbys; a takeover of these LUNs opens them cold meanwhile.") + logger.warning(msg) + try: + ec.log_event_cluster( + cluster_id=cluster.get_id(), + domain=ec.DOMAIN_CLUSTER, + event=ec.EVENT_LUN_TAKEOVER, + db_object=node, + caused_by=ec.CAUSED_BY_MONITOR, + message=msg, + node_id=node_id, + event_level=EventObj.LEVEL_WARN) + except Exception as e: # noqa: BLE001 + logger.error("Warm standby budget event of %s not logged: %s", node_id, e) diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index e6b2573fdb..fcf909b50f 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -230,6 +230,12 @@ def is_topology_owned(self) -> bool: # Requires raid_devices. Gates every multi-attach path (device NQN, # controller-ID ranges, hold timeouts, owner loopback). multi_attach: bool = False + # Multi-attach clusters, opt-in: keep a warm standby alceml (standby=true, + # page map kept warm in hugepages) of every LUN on each attached + # non-serving node, so a takeover activates it on the first I/O instead of + # loading the whole page map cold (lun_takeover.ensure_warm_standbys). + # Costs the page-map memory of every such LUN on each standby node. + multi_attach_warm_standby: bool = False # Multi-attach clusters: the storage array the LUNs live on. A LUN is # taken over by another node only after its serving node is fenced at the # array, through the plugin named here (entry point in the group diff --git a/simplyblock_core/multi_attach.py b/simplyblock_core/multi_attach.py index 756ac8c65f..f7fcbc378e 100644 --- a/simplyblock_core/multi_attach.py +++ b/simplyblock_core/multi_attach.py @@ -187,6 +187,18 @@ def reservations_enabled(cluster, device) -> bool: return is_multi_attach(device) and getattr(device, "lun_reservations", False) is True +def warm_standby_enabled(cluster, device) -> bool: + """Whether ``device`` gets warm standby alcemls on its attached + non-serving nodes (lun_takeover.ensure_warm_standbys): a multi-attach + cluster with ``multi_attach_warm_standby`` set, and a device whose + alceml instances run with reservation keys (a standby activates by + preempting the old holder's reservation; without one nothing would fence + the old holder on activation).""" + return (is_multi_attach(cluster) + and getattr(cluster, "multi_attach_warm_standby", False) is True + and reservations_enabled(cluster, device)) + + def alceml_create_kwargs(device, node=None, cluster=None) -> dict: """Extra ``bdev_alceml_create`` kwargs for a multi-attach device's alceml on ``node``: the stuck-I/O timeout, and the reservation key when @@ -324,6 +336,44 @@ def served_by_online_node(owner, nodes_by_id) -> list: return out +#: Device states in which a warm standby is kept (any other: removed). +_WARM_STANDBY_DEVICE_STATES = ("online",) +#: Device states whose LUN still counts in a node's warm-standby memory +#: budget (a LUN unavailable for a while gets its standby back). +_WARM_STANDBY_BUDGET_STATES = ("online", "unavailable", "read_only", "cannot_allocate") + + +def warm_standby_devices(cluster, node_id, nodes, statuses=_WARM_STANDBY_DEVICE_STATES) -> list: + """``(owner, device)`` of every LUN ``node_id`` keeps a warm standby of + (Cluster.multi_attach_warm_standby): ``warm_standby_enabled`` for the + device, the device in one of ``statuses``, the node among the LUN's + attached hosts (array_controller.desired_hosts: the owner, its LVS + secondary and tertiary), not the node serving it and not fenced from it. + Empty on any other cluster.""" + if not (is_multi_attach(cluster) and getattr(cluster, "multi_attach_warm_standby", False) is True): + return [] + from simplyblock_core.controllers.array_controller import desired_hosts + by_id = {n.get_id(): n for n in nodes} + out = [] + for owner in nodes: + for dev in getattr(owner, "nvme_devices", None) or []: + if not warm_standby_enabled(cluster, dev) or dev.status not in statuses: + continue + if dev.serving_node() == node_id or fenced_from(dev, node_id): + continue + if node_id in desired_hosts(owner, dev, by_id): + out.append((owner, dev)) + return out + + +def warm_standby_capacity(cluster, node_id, nodes) -> int: + """Raw capacity (bytes) of the LUNs ``node_id`` keeps (or will keep, once + they are back online) a warm standby of: the capacity whose alceml page + map the node holds on top of its own devices. 0 on any other cluster.""" + return sum(max(int(dev.size or 0), 0) for _, dev in + warm_standby_devices(cluster, node_id, nodes, _WARM_STANDBY_BUDGET_STATES)) + + def normalize_wwn(value) -> str: # lsblk reports a SCSI WWN as 0x, arrays as naa.; the same id. v = (value or "").strip().lower() diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 6de3b0a8e2..30119716b1 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -981,7 +981,7 @@ def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3, write_protection=False, full_page_unmap=False, checksum_method=0, cache_size=0, cache_eviction_threshold=0, force_4k_atomic=False, stuck_io_timeout_ms=0, - reservation_key=0, standby=False): + reservation_key=0, standby=False, refresh_interval_ms=0): params = { "name": alceml_name, "cntr_path": nvme_name, @@ -1031,6 +1031,10 @@ def bdev_alceml_create(self, alceml_name, nvme_name, uuid, pba_init_mode=3, params["reservation_key"] = int(reservation_key) if standby: params["standby"] = True + # Warm standby: how often its page map is refreshed from the + # LUN's region counters (data-plane default 2000 ms when unset). + if refresh_interval_ms: + params["refresh_interval_ms"] = int(refresh_interval_ms) return self._request("bdev_alceml_create", **params) def alceml_get_role(self, name): diff --git a/simplyblock_core/services/storage_node_monitor.py b/simplyblock_core/services/storage_node_monitor.py index d5bbf5dfcd..e7a8e231e0 100644 --- a/simplyblock_core/services/storage_node_monitor.py +++ b/simplyblock_core/services/storage_node_monitor.py @@ -2966,6 +2966,7 @@ def check_node(snode): if snode.status == StorageNode.STATUS_ONLINE: storage_network.publish_ports(snode, at_most_every=PORT_PUBLISH_INTERVAL_SEC) _reintegrate_fenced_luns(snode) + _ensure_warm_standbys(snode) # Self-heal a node stranded in IN_SHUTDOWN. A completed shutdown flips the # node to OFFLINE; if it is still IN_SHUTDOWN well past the grace window the @@ -3311,6 +3312,41 @@ def _run(): t.start() +_warm_standby_last: dict = {} +_warm_standby_threads: dict = {} + + +def _ensure_warm_standbys(snode): + """Warm-standby multi-attach clusters (only, Cluster.multi_attach_warm_standby): + an ONLINE node keeps a standby alceml of every LUN it is an attached + standby of, and closes the ones it is no longer a standby of + (lun_takeover.ensure_warm_standbys), at most every + MULTI_ATTACH_WARM_STANDBY_INTERVAL_SEC per node, in its own thread.""" + try: + cluster = db.get_cluster_by_id(snode.cluster_id) + except Exception as e: # noqa: BLE001 + logger.debug("Warm standbys of %s: cluster not read: %s", snode.get_id(), e) + return + if not (multi_attach.is_multi_attach(cluster) + and getattr(cluster, "multi_attach_warm_standby", False) is True): + return + node_id = snode.get_id() + now = time.monotonic() + if now - _warm_standby_last.get(node_id, 0) < constants.MULTI_ATTACH_WARM_STANDBY_INTERVAL_SEC: + return + t = _warm_standby_threads.get(node_id) + if t is not None and t.is_alive(): + return + _warm_standby_last[node_id] = now + + def _run(): + from simplyblock_core.controllers import lun_takeover + lun_takeover.ensure_warm_standbys(snode) + t = threading.Thread(target=_run, name=f"warm-standby-{node_id[:8]}", daemon=True) + _warm_standby_threads[node_id] = t + t.start() + + def readmit_devices_after_node_online(node_id): """Re-admit a recovered node's devices right after it was set ONLINE. diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index ac729fb223..f9629cb34b 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4414,6 +4414,14 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, "devices visible (unbind them from SPDK) or pass spdk_sys_mem.", ssd_pcis, node_addr) return False minimum_sys_memory = max(minimum_sys_memory, utils.sys_memory_for_capacity(capacity)) + # Warm-standby multi-attach clusters (Cluster.multi_attach_warm_standby): + # the node will also hold the page map of every LUN it becomes a warm + # standby of, i.e. sys_memory_for_capacity(own + standby LUN capacity). + # That set (the LUNs of the nodes whose LVS secondary/tertiary this node + # becomes) is not known yet here; it is counted from the node's first + # restart on (_warm_standby_capacity), and until then + # lun_takeover.ensure_warm_standbys opens no standby beyond this budget + # and raises a cluster event instead. max_lvol = node_config.get("max_lvol") # Clamp rather than fail: a config file generated before the cap was # enforced (or hand-edited) can carry a larger value, and refusing the @@ -8910,6 +8918,21 @@ def _finalize_node_removal(removed_node: StorageNode): logger.info("done") +def _warm_standby_capacity(db_controller, snode) -> int: + """Raw capacity of the LUNs ``snode`` keeps warm standbys of + (multi_attach.warm_standby_capacity); 0 when unknown or on any cluster + without Cluster.multi_attach_warm_standby.""" + try: + cluster = db_controller.get_cluster_by_id(snode.cluster_id) + if not multi_attach.is_multi_attach(cluster): + return 0 + return multi_attach.warm_standby_capacity( + cluster, snode.get_id(), db_controller.get_storage_nodes_by_cluster_id(snode.cluster_id)) + except Exception as e: # noqa: BLE001 + logger.warning("Warm standby capacity of %s unknown: %s", snode.get_id(), e) + return 0 + + def restart_storage_node( node_id, max_lvol=0, max_snap=0, max_prov=0, spdk_image=None, set_spdk_debug=None, @@ -9701,6 +9724,10 @@ def _restart_storage_node_impl( # The SPDK container's memory limit: recomputed from the node's devices # (the stored figure may come from an older formula), else the stored one. raw_capacity = sum(max(int(d.size or 0), 0) for d in (snode.nvme_devices or [])) + # Warm-standby multi-attach clusters: the node also holds the page map of + # every LUN it keeps a warm standby of (lun_takeover.ensure_warm_standbys), + # so those LUNs count as its own capacity. 0 on any other cluster. + raw_capacity += _warm_standby_capacity(db_controller, snode) minimum_sys_memory = (utils.sys_memory_for_capacity(raw_capacity) if raw_capacity else snode.minimum_sys_memory or 0) snode.spdk_mem = minimum_hp_memory diff --git a/simplyblock_web/api/v1/cluster.py b/simplyblock_web/api/v1/cluster.py index db7d3ad962..5a33e9f9ae 100644 --- a/simplyblock_web/api/v1/cluster.py +++ b/simplyblock_web/api/v1/cluster.py @@ -73,6 +73,7 @@ def add_cluster(): raid_devices=raid_devices, multi_attach=multi_attach, ftt=ftt, + multi_attach_warm_standby=bool(cl_data.get('multi_attach_warm_standby', False)), kms_settings=cl_data.get('kms_settings'), lvol_geometry_enabled=lvol_geometry_enabled, # Absent inherits the prefix of a cluster already on this control plane. diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index 2b918497de..cfce8359e1 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -214,6 +214,7 @@ class ClusterDTO(BaseModel): md_journal: bool = False raid_devices: bool = False multi_attach: bool = False + multi_attach_warm_standby: bool = False secret: SecretStr tls_enabled: bool max_fault_tolerance: int @@ -268,6 +269,7 @@ def from_model(model: Cluster, stat_obj: StatsObject | None = None): md_journal=model.md_journal, raid_devices=model.raid_devices, multi_attach=model.multi_attach, + multi_attach_warm_standby=getattr(model, "multi_attach_warm_standby", False) is True, secret=model.secret, tls_enabled=model.tls, max_fault_tolerance=model.max_fault_tolerance, diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index 7bdef688a8..736f5a4a4d 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -86,6 +86,9 @@ class ClusterParams(BaseModel): multi_attach: bool = False #: Node failures to tolerate in multi-attach mode; only with multi_attach. ftt: Literal[1, 2] | None = None + #: Multi-attach only: keep warm standby alcemls of every LUN on its + #: attached non-serving nodes (fast takeover; costs page-map hugepages). + multi_attach_warm_standby: bool = False is_single_node: bool = False fabric: str = "tcp" cr_name: str = "" diff --git a/tests/unit/test_lun_warm_standby.py b/tests/unit/test_lun_warm_standby.py new file mode 100644 index 0000000000..bec815a796 --- /dev/null +++ b/tests/unit/test_lun_warm_standby.py @@ -0,0 +1,476 @@ +"""Multi-attach warm standby (Cluster.multi_attach_warm_standby, opt-in). + + - lun_takeover.ensure_warm_standbys: a standby alceml (standby=true, + open-only, the reservation key) on attached non-serving nodes only; never + exported; never on fenced / serving / unattached nodes or on clusters + without the flag; closed when the node is no longer a standby; refused + beyond the node's memory budget, with a cluster event. + - the takeover reuses a warm standby: no alceml create, the export is + built, the activation is awaited after the path switch; a standby that + does not activate falls back to the array fence and a cold open. + - memory: warm-standby LUNs count in the node's restart budget. + - plumbing: cluster flag, monitor driver. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import ( + cluster_ops, + constants, + multi_attach, + storage_node_ops, + utils, +) +from simplyblock_core.array.fake import FakeArrayPlugin +from simplyblock_core.controllers import device_controller, lun_takeover +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import storage_node_monitor +from tests.unit.test_lun_takeover import BY_ID, DEV_ID, SIZE, WWN, _bed, _TakeoverCase + +ALCEML = f"alceml_{DEV_ID}" +BASE = utils.aio_bdev_name_for_serial(WWN) + + +def _warm_bed(flag=True, reservations=True): + """_bed with the owner node-a ONLINE and serving its LUN; node-b / node-c + its LVS secondary / tertiary (attached standbys), node-d a peer.""" + db, dev = _bed() + db.cluster.multi_attach_warm_standby = flag + dev.lun_reservations = reservations + db.nodes["node-a"].status = StorageNode.STATUS_ONLINE + return db, dev + + +def _role(spdk, fn): + spdk.alceml_get_role = fn + + +def _standby_role(_name): + return {"role": "standby", "state": "standby", "standby_at_open": True, "warm": True, + "refreshes_count": 7} + + +class _WarmCase(_TakeoverCase): + + def ensure(self, db, nid): + self.m_DBController.return_value = db + return lun_takeover.ensure_warm_standbys(db.nodes[nid]) + + +# ------------------------------------------------------ keeping standbys + +class TestEnsureWarmStandbys(_WarmCase): + + def test_created_on_attached_non_serving_nodes_only(self): + db, _ = _warm_bed() + for nid in ("node-a", "node-b", "node-c", "node-d"): + self.ensure(db, nid) + for nid in ("node-b", "node-c"): + spdk = db.nodes[nid].spdk + calls = spdk.ops("bdev_alceml_create") + self.assertEqual(len(calls), 1, nid) + kw = calls[0][2] + self.assertEqual(calls[0][1][:3], (ALCEML, BASE, DEV_ID)) + self.assertIs(kw["standby"], True) + self.assertEqual(kw["pba_init_mode"], lun_takeover.ALCEML_OPEN_ONLY) + self.assertEqual(kw["reservation_key"], multi_attach.reservation_key(db.nodes[nid])) + self.assertEqual(kw["refresh_interval_ms"], constants.MULTI_ATTACH_WARM_REFRESH_MS) + self.assertEqual(kw["stuck_io_timeout_ms"], constants.MULTI_ATTACH_ALCEML_STUCK_IO_MS) + # Over the node's own path to the LUN (its agent's lun_info). + self.assertEqual(spdk.ops("bdev_aio_create")[0][1][:2], (BASE, BY_ID)) + # Never exported: a peer's I/O would activate it. + self.assertEqual(spdk.ops("bdev_PT_NoExcl_create"), []) + self.assertEqual(spdk.ops("subsystem_create"), []) + self.assertEqual(spdk.ops("nvmf_subsystem_add_ns"), []) + # The serving node and an unattached peer get nothing. + self.assertEqual(db.nodes["node-a"].spdk.calls, []) + self.assertEqual(db.nodes["node-d"].spdk.calls, []) + db.nodes["node-d"].agent.lun_info.assert_not_called() + + def test_idempotent(self): + db, _ = _warm_bed() + self.assertEqual(self.ensure(db, "node-b")["created"], [DEV_ID]) + _role(db.nodes["node-b"].spdk, _standby_role) + self.assertEqual(self.ensure(db, "node-b"), {"created": [], "removed": [], "over_budget": []}) + self.assertEqual(len(db.nodes["node-b"].spdk.ops("bdev_alceml_create")), 1) + + def test_not_on_a_fenced_node(self): + db, dev = _warm_bed() + dev.fenced_node_ids = ["node-c"] + self.ensure(db, "node-c") + self.assertEqual(db.nodes["node-c"].spdk.calls, []) + + def test_not_on_the_serving_node_after_a_takeover(self): + db, dev = _warm_bed() + dev.serving_node_id = "node-b" + dev.fenced_node_ids = ["node-a"] + self.ensure(db, "node-b") + self.assertEqual(db.nodes["node-b"].spdk.ops("bdev_alceml_create"), []) + # The owner, fenced, gets none either; the tertiary does. + self.ensure(db, "node-a") + self.assertEqual(db.nodes["node-a"].spdk.ops("bdev_alceml_create"), []) + self.ensure(db, "node-c") + self.assertEqual(len(db.nodes["node-c"].spdk.ops("bdev_alceml_create")), 1) + + def test_not_on_offline_node_or_device(self): + db, dev = _warm_bed() + db.nodes["node-b"].status = StorageNode.STATUS_OFFLINE + self.ensure(db, "node-b") + self.assertEqual(db.nodes["node-b"].spdk.calls, []) + db.nodes["node-b"].status = StorageNode.STATUS_ONLINE + dev.status = "unavailable" + self.ensure(db, "node-b") + self.assertEqual(db.nodes["node-b"].spdk.ops("bdev_alceml_create"), []) + + def test_not_without_the_flag_or_reservations(self): + for flag, res in ((False, True), (True, False)): + with self.subTest(flag=flag, reservations=res): + db, _ = _warm_bed(flag=flag, reservations=res) + for nid in db.nodes: + self.ensure(db, nid) + self.assertEqual(db.nodes[nid].spdk.calls, []) + db.nodes[nid].agent.lun_info.assert_not_called() + + def test_not_while_a_takeover_of_the_lun_runs(self): + db, _ = _warm_bed() + lock = lun_takeover._local_lock(DEV_ID) + with lock: + self.ensure(db, "node-b") + self.assertEqual(db.nodes["node-b"].spdk.ops("bdev_alceml_create"), []) + + def test_lun_not_visible_opens_nothing(self): + db, _ = _warm_bed() + db.nodes["node-b"].agent.lun_info.return_value = ([], None) + self.assertEqual(self.ensure(db, "node-b")["created"], []) + self.assertEqual(db.nodes["node-b"].spdk.calls, []) + + def test_failed_open_closes_the_base_bdev(self): + db, _ = _warm_bed() + b = db.nodes["node-b"] + b.spdk.fail.add("bdev_alceml_create") + self.assertEqual(self.ensure(db, "node-b")["created"], []) + self.assertNotIn(BASE, b.spdk.bdevs) + self.assertTrue(b.spdk.ops("bdev_aio_delete")) + + def test_removed_standby_role_cleans_up(self): + db, dev = _warm_bed() + c = db.nodes["node-c"] + self.ensure(db, "node-c") + self.assertIn(ALCEML, c.spdk.bdevs) + _role(c.spdk, _standby_role) + # node-c is no longer the owner's tertiary: not attached any more. + db.nodes["node-a"].tertiary_node_id = "" + report = self.ensure(db, "node-c") + self.assertEqual(report["removed"], [DEV_ID]) + self.assertNotIn(ALCEML, c.spdk.bdevs) + self.assertNotIn(BASE, c.spdk.bdevs) + deletes = [op for op, *_ in c.spdk.calls if op.endswith("_delete")] + self.assertEqual(deletes, ["bdev_alceml_delete", "bdev_aio_delete"]) + + def test_fenced_standby_is_closed(self): + db, dev = _warm_bed() + b = db.nodes["node-b"] + self.ensure(db, "node-b") + _role(b.spdk, _standby_role) + dev.fenced_node_ids = ["node-b"] + self.assertEqual(self.ensure(db, "node-b")["removed"], [DEV_ID]) + self.assertNotIn(ALCEML, b.spdk.bdevs) + + def test_exported_or_active_stack_is_never_closed(self): + db, dev = _warm_bed() + b = db.nodes["node-b"] + dev.fenced_node_ids = ["node-b"] + b.spdk.bdevs[ALCEML] = {"name": ALCEML} + _role(b.spdk, lambda _n: {"role": "active", "state": "active"}) + self.assertEqual(self.ensure(db, "node-b")["removed"], []) + _role(b.spdk, _standby_role) + b.spdk.bdevs[f"{ALCEML}_PT"] = {"name": f"{ALCEML}_PT"} + self.assertEqual(self.ensure(db, "node-b")["removed"], []) + self.assertIn(ALCEML, b.spdk.bdevs) + + def test_owner_standby_keeps_its_base_bdev(self): + # The owner, no longer serving and not a standby: its alceml closes, + # its own base bdev stays (its restart path owns it). + db, dev = _warm_bed() + a = db.nodes["node-a"] + dev.serving_node_id = "node-b" + dev.fenced_node_ids = ["node-a"] + a.spdk.bdevs[BASE] = {"name": BASE} + a.spdk.bdevs[ALCEML] = {"name": ALCEML} + _role(a.spdk, _standby_role) + self.assertEqual(self.ensure(db, "node-a")["removed"], [DEV_ID]) + self.assertNotIn(ALCEML, a.spdk.bdevs) + self.assertIn(BASE, a.spdk.bdevs) + + +class TestRetireKeepsWarmStandby(_WarmCase): + """retire_stale_local_stack (node back ONLINE without a restart) keeps + the warm standby of a LUN served elsewhere; a fenced node's is retired.""" + + def _owner_standby(self): + db, dev = _warm_bed() + a = db.nodes["node-a"] + dev.serving_node_id = "node-b" + a.spdk.bdevs[ALCEML] = {"name": ALCEML} + _role(a.spdk, _standby_role) + p = patch("simplyblock_core.db_controller.DBController", return_value=db) + p.start() + self.addCleanup(p.stop) + return db, dev, a + + def test_kept(self): + _db, _dev, a = self._owner_standby() + self.assertEqual(lun_takeover.retire_stale_local_stack(a), []) + self.assertIn(ALCEML, a.spdk.bdevs) + + def test_retired_when_fenced_or_deposed(self): + _db, dev, a = self._owner_standby() + dev.fenced_node_ids = ["node-a"] + self.assertEqual(lun_takeover.retire_stale_local_stack(a), [DEV_ID]) + self.assertNotIn(ALCEML, a.spdk.bdevs) + # A deposed former active (not opened standby) is stale as well. + _db, dev, a = self._owner_standby() + _role(a.spdk, lambda _n: {"role": "standby", "state": "deposed", "standby_at_open": False}) + self.assertEqual(lun_takeover.retire_stale_local_stack(a), [DEV_ID]) + + +# --------------------------------------------------------------- memory + +class TestWarmStandbyMemory(_WarmCase): + + def test_over_budget_is_refused_with_one_event(self): + db, _ = _warm_bed() + b = db.nodes["node-b"] + b.minimum_sys_memory = utils.sys_memory_for_capacity(0) # no room for a page map + report = self.ensure(db, "node-b") + self.assertEqual(report["over_budget"], [DEV_ID]) + self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) + budget_events = [e for e in self.events if "memory budget" in e["message"]] + self.assertEqual(len(budget_events), 1) + self.assertEqual(budget_events[0]["node_id"], "node-b") + # Not repeated while nothing changes. + self.ensure(db, "node-b") + self.assertEqual(len([e for e in self.events if "memory budget" in e["message"]]), 1) + + def test_within_budget_is_opened(self): + db, _ = _warm_bed() + b = db.nodes["node-b"] + b.minimum_sys_memory = utils.sys_memory_for_capacity(SIZE) + self.assertEqual(self.ensure(db, "node-b")["created"], [DEV_ID]) + self.assertFalse([e for e in self.events if "memory budget" in e["message"]]) + + def test_restart_budget_counts_warm_standby_luns(self): + db, dev = _warm_bed() + b, d = db.nodes["node-b"], db.nodes["node-d"] + self.assertEqual(multi_attach.warm_standby_capacity(db.cluster, "node-b", list(db.nodes.values())), + SIZE) + self.assertEqual(storage_node_ops._warm_standby_capacity(db, b), SIZE) + # A device unavailable for a while still counts: its standby returns. + dev.status = "unavailable" + self.assertEqual(storage_node_ops._warm_standby_capacity(db, b), SIZE) + # Not a standby (unattached peer, serving node), not a warm cluster. + self.assertEqual(storage_node_ops._warm_standby_capacity(db, d), 0) + self.assertEqual(storage_node_ops._warm_standby_capacity(db, db.nodes["node-a"]), 0) + db.cluster.multi_attach_warm_standby = False + self.assertEqual(storage_node_ops._warm_standby_capacity(db, b), 0) + + +# ------------------------------------------------------------- takeover + +class TestWarmTakeover(_WarmCase): + + def setUp(self): + super().setUp() + p = patch.object(device_controller, "device_set_unavailable", return_value=True) + self.m_unavailable = p.start() + self.addCleanup(p.stop) + + def _with_standby(self): + """The secondary holds a warm standby (opened by ensure_warm_standbys); + then the serving owner fails.""" + db, dev = _warm_bed() + self.ensure(db, "node-b") + b = db.nodes["node-b"] + self.assertIn(ALCEML, b.spdk.bdevs) + b.spdk.calls.clear() + db.nodes["node-a"].status = StorageNode.STATUS_OFFLINE + return db, dev, b + + def test_reuses_the_warm_standby(self): + db, _, b = self._with_standby() + seen = [] + + def role(_name): + # The peers' I/O reaches it once the paths are switched. + switched = bool(b.spdk.ops("attach")) + seen.append(switched) + if switched: + return {"role": "active", "state": "active", "epoch": 3} + return _standby_role(_name) + _role(b.spdk, role) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) + self.assertEqual(b.spdk.ops("bdev_alceml_delete"), []) + self.assertEqual(len(b.spdk.ops("bdev_PT_NoExcl_create")), 1) + self.assertEqual(len(b.spdk.ops("subsystem_create")), 1) + self.assertEqual(len(b.spdk.ops("nvmf_subsystem_add_ns")), 1) + # Checked once before the export (standby), then awaited after the + # path switch until active. + self.assertEqual(seen[0], False) + self.assertEqual(seen[-1], True) + dev = self.dev(db) + self.assertEqual(dev.serving_node(), "node-b") + self.assertIn("node-a", dev.fenced_node_ids) + self.m_unavailable.assert_not_called() + + def test_activation_failure_falls_back_to_a_cold_open(self): + db, _, b = self._with_standby() + + def role(_name): + # The standby never activates; the instance reopened active does. + if b.spdk.ops("bdev_alceml_create"): + return {"role": "active", "state": "active"} + return _standby_role(_name) + _role(b.spdk, role) + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 0.2): + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + # The failed node is fenced at the array (synchronously). + self.assertIn(("fence", WWN, "host-node-a"), FakeArrayPlugin.calls) + # Export torn down, standby closed, then reopened active with the key. + ops = [op for op, *_ in b.spdk.calls] + self.assertLess(ops.index("attach"), ops.index("subsystem_delete")) + self.assertLess(ops.index("bdev_alceml_delete"), ops.index("bdev_alceml_create")) + create = b.spdk.ops("bdev_alceml_create") + self.assertEqual(len(create), 1) + self.assertNotIn("standby", create[0][2]) + self.assertEqual(create[0][2]["reservation_key"], multi_attach.reservation_key(b)) + self.assertEqual(create[0][2]["pba_init_mode"], lun_takeover.ALCEML_OPEN_ONLY) + self.assertEqual(len(b.spdk.ops("subsystem_create")), 2) + self.assertIn(BASE, b.spdk.bdevs) + dev = self.dev(db) + self.assertEqual(dev.serving_node(), "node-b") + self.assertIn("node-a", dev.fenced_node_ids) + self.assertTrue(any("did not activate" in e["message"] for e in self.events)) + self.m_unavailable.assert_not_called() + + def test_cold_fallback_failure_sets_the_device_unavailable(self): + db, _, b = self._with_standby() + _role(b.spdk, _standby_role) + b.spdk.fail.add("bdev_alceml_create") + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 0.2): + self.assertEqual(self.run_takeover(db), lun_takeover.FAILED) + self.m_unavailable.assert_called_once_with(DEV_ID) + self.assertTrue(any("cold open failed" in e["message"] for e in self.events)) + + def test_standby_whose_activation_failed_is_reopened(self): + db, _, b = self._with_standby() + + def role(_name): + if b.spdk.ops("bdev_alceml_create"): + return {"role": "active", "state": "active"} + return {"role": "standby", "state": "act_failed"} + _role(b.spdk, role) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(len(b.spdk.ops("bdev_alceml_delete")), 1) + create = b.spdk.ops("bdev_alceml_create") + self.assertEqual(len(create), 1) + self.assertNotIn("standby", create[0][2]) + + def test_unresponsive_warm_standby_moves_to_the_next(self): + db, _, b = self._with_standby() + c = db.nodes["node-c"] + + def b_dies(_name): + b.spdk.down = True + raise ConnectionError("gone") + _role(b.spdk, b_dies) + _role(c.spdk, lambda _n: {"role": "active", "state": "active"}) + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 0.2): + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + + def test_without_the_flag_an_existing_standby_is_not_reused(self): + # Flag off: the regular path (wait for the reservation before the export). + db, _, b = self._with_standby() + db.cluster.multi_attach_warm_standby = False + _role(b.spdk, lambda _n: {"role": "active", "state": "active"}) + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(b.spdk.ops("bdev_alceml_create"), []) + self.assertEqual(self.dev(db).serving_node(), "node-b") + + +# ------------------------------------------------------------- plumbing + +class TestWarmStandbyPlumbing(unittest.TestCase): + + def test_helper(self): + db, dev = _warm_bed() + self.assertTrue(multi_attach.warm_standby_enabled(db.cluster, dev)) + db.cluster.multi_attach_warm_standby = False + self.assertFalse(multi_attach.warm_standby_enabled(db.cluster, dev)) + db.cluster.multi_attach_warm_standby = True + dev.lun_reservations = False + self.assertFalse(multi_attach.warm_standby_enabled(db.cluster, dev)) + self.assertFalse(multi_attach.warm_standby_enabled(Cluster(), dev)) + + def test_cluster_flag_needs_multi_attach(self): + self.assertFalse(Cluster().multi_attach_warm_standby) + with self.assertRaises(ValueError): + cluster_ops._validate_warm_standby(False, True) + cluster_ops._validate_warm_standby(True, True) + cluster_ops._validate_warm_standby(False, False) + + def test_cli_flag(self): + import argparse + + from simplyblock_cli.cli import CLIWrapper + cli = CLIWrapper.__new__(CLIWrapper) + cli.developer_mode = False + for cmd in ("create", "add"): + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="c") + with patch.object(CLIWrapper, "add_sub_command", + lambda self, subp, name, help, **k: subp.add_parser(name)): + getattr(cli, f"init_cluster__{cmd}")(sub) + args, _ = parser.parse_known_args([cmd, "--multi-attach-warm-standby"]) + self.assertTrue(args.multi_attach_warm_standby, cmd) + + def test_rpc_sends_refresh_interval_for_a_standby_only(self): + from simplyblock_core.rpc_client import RPCClient + rpc = RPCClient.__new__(RPCClient) + rpc._request = MagicMock(return_value=True) + rpc.bdev_alceml_create("a", "b", "u", pba_init_mode=1, reservation_key=1 << 32, + standby=True, refresh_interval_ms=2000) + params = rpc._request.call_args.kwargs + self.assertEqual((params["standby"], params["refresh_interval_ms"]), (True, 2000)) + rpc.bdev_alceml_create("a", "b", "u", pba_init_mode=1, refresh_interval_ms=2000) + params = rpc._request.call_args.kwargs + self.assertNotIn("standby", params) + self.assertNotIn("refresh_interval_ms", params) + + def test_monitor_drives_warm_clusters_only(self): + db, _ = _warm_bed() + node = db.nodes["node-b"] + storage_node_monitor._warm_standby_last.clear() + storage_node_monitor._warm_standby_threads.clear() + with patch.object(storage_node_monitor, "db") as mdb, \ + patch.object(lun_takeover, "ensure_warm_standbys") as ensure: + mdb.get_cluster_by_id.return_value = db.cluster + storage_node_monitor._ensure_warm_standbys(node) + storage_node_monitor._warm_standby_threads["node-b"].join(5) + ensure.assert_called_once_with(node) + # Throttled per node. + storage_node_monitor._ensure_warm_standbys(node) + self.assertEqual(ensure.call_count, 1) + storage_node_monitor._warm_standby_last.clear() + db.cluster.multi_attach_warm_standby = False + storage_node_monitor._ensure_warm_standbys(node) + self.assertEqual(ensure.call_count, 1) + storage_node_monitor._warm_standby_last.clear() + storage_node_monitor._warm_standby_threads.clear() + + +if __name__ == "__main__": + unittest.main() From 1012523c6ac16cc321db0456c79435a6f4b8482c Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 20:50:25 +0100 Subject: [PATCH 41/50] =?UTF-8?q?feat(array):=20journal=20placement=20?= =?UTF-8?q?=E2=80=94=20a=20LUN=20of=20its=20own,=20or=20a=20local=20disk?= =?UTF-8?q?=20or=20partition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node's journal on a multi-attach cluster whose array plugin creates the nodes' LUNs is now placed by the array registration: - journal "lun": node add creates a journal LUN next to the data LUN, with journal_options applied over the array's options (e.g. a pool of higher performance or priority) and journal_lun_size (default 3% of lun_size, at least 2 GiB, whole GiB). It is presented to that node only, never multi-attached, recorded as StorageNode.array_journal_lun_id, detached on node delete and deleted with the cluster. The host needs no local device: sn configure --lblk-array-only. - journal "local" (default, unchanged for existing beds): one local disk or partition named with --lblk-journal-only, now a partition as well. register_array / POST /clusters/{id}/arrays / cluster set-array gain journal, journal_lun_size, journal_options; the array view reports them. A journal partition on the data LUN is not offered: takeover and growth would have to address a partition of a multi-attached LUN. Co-Authored-By: Claude Opus 5.5 --- simplyblock_cli/cli-reference.yaml | 33 +- simplyblock_cli/cli.py | 6 +- simplyblock_cli/clibase.py | 24 +- simplyblock_core/array/__init__.py | 7 +- .../controllers/array_controller.py | 141 +++++++-- simplyblock_core/models/cluster.py | 14 + simplyblock_core/models/storage_node.py | 4 + simplyblock_core/storage_node_ops.py | 42 ++- simplyblock_core/utils/__init__.py | 57 +++- simplyblock_web/api/v2/cluster/array.py | 16 +- simplyblock_web/node_configure.py | 18 +- tests/unit/test_array_journal.py | 297 ++++++++++++++++++ tests/unit/test_array_lifecycle.py | 3 +- tests/unit/test_lblk_journal_only.py | 10 +- 14 files changed, 605 insertions(+), 67 deletions(-) create mode 100644 tests/unit/test_array_journal.py diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 31ec721bbe..8d99bf5d3b 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -116,10 +116,15 @@ commands: type: str default: "" - name: "--lblk-journal-only" - help: "With --lblk on a multi-attach cluster whose array plugin creates each node's data LUN at add-node: the one selected whole disk is the node's journal and nothing else. It is never partitioned, and the node has no local data device; its data device is the array LUN. Needs exactly one disk, named with --blk-names or --blk-serials." + help: "With --lblk on a multi-attach cluster whose array plugin creates each node's data LUN at add-node and keeps its journal locally: the one selected disk or partition is the node's journal and nothing else. It is used whole (a disk is never partitioned), and the node has no local data device; its data device is the array LUN. Needs exactly one disk or partition, named with --blk-names or --blk-serials." dest: lblk_journal_only type: bool action: store_true + - name: "--lblk-array-only" + help: "With --lblk on a multi-attach cluster whose array plugin creates each node's LUNs at add-node and keeps its journal on a LUN: the node uses no local device. Its journal LUN and data LUN are created on the array when the node is added. Not with --blk-names, --blk-names-exclude, --blk-serials, --lblk-journal-only or --lblk-journal-device." + dest: lblk_array_only + type: bool + action: store_true - name: "--lblk-journal-device" help: "With --lblk and whole disks: dedicate each node's smallest disk to the journal instead of partitioning it into a journal partition and a data partition (the default). Needs at least two disks per node. Partition selections always carry their journal in a split partition." dest: lblk_journal_device @@ -1614,7 +1619,10 @@ commands: the array plugin registered here. The plugin is loaded and connected with the credentials before anything is saved. The credentials are wrapped by the cluster's KMS (or stay in an external Vault KV / KMIP secret named by --credential-ref), never stored in the clear. With --lun-size, - every node added afterwards gets a LUN of that size created and attached on the array. + every node added afterwards gets a LUN of that size created and attached on the array. Its journal is + kept on a local disk or partition of its host (--journal local, the default: configure the host with + --lblk-journal-only) or on a LUN of its own created next to it (--journal lun: configure the host with + --lblk-array-only), for example in a pool of higher performance named by --journal-options. arguments: - name: "cluster_id" help: "The cluster id." @@ -1652,6 +1660,27 @@ commands: dest: lun_size type: size default: "0" + - name: "--journal" + help: > + Where each node's journal is kept: `local` (a disk or partition of its host) or `lun` (a LUN of its own, + created on the array next to the node's data LUN; needs --lun-size). Default: `local`. + dest: journal + type: str + default: "local" + choices: + - "local" + - "lun" + - name: "--journal-lun-size" + help: "With --journal lun: size of each node's journal LUN, e.g. `64G`. Default: 3% of --lun-size, at least 2 GiB." + dest: journal_lun_size + type: size + default: "0" + - name: "--journal-options" + help: > + With --journal lun: plugin options for the journal LUNs as a JSON object, applied over --options, e.g. + `{"pool": "fast-pool"}` to create them in a pool of higher performance or priority. + dest: journal_options + type: str - name: get-array help: "Shows the storage array registered for a multi-attach cluster (without its credentials)." arguments: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 61259a2cd8..1f05e9705b 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -112,7 +112,8 @@ def init_storage_node__configure(self, subparser): subcommand.add_argument('--blk-names', help='Comma separated list of block devices to use (requires --lblk). Each entry is a persistent /dev/disk name, such as /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_drive-scsi0 or /dev/disk/by-partuuid/28427de0-1916-4c05-895f-0829cd8790ba. A kernel name (sdb) or kernel path (/dev/sdb) is refused: it states a position in this boot\'s enumeration order, and it would name another disk after a reboot that probes the controllers in another order. The path is taken as given: the node records it, and add-node and every restart open the device by that path alone, never by serial or kernel name. A path that no longer exists is a missing device. Use --blk-serials for a device udev published no persistent name for. Requested devices must be eligible; a busy device is an error.', type=str, default='', dest='blk_names', required=False) subcommand.add_argument('--blk-names-exclude', help='Comma separated list of block devices to exclude (requires --lblk), spelled the way --blk-names takes them. All other eligible disks are used.', type=str, default='', dest='blk_names_exclude', required=False) subcommand.add_argument('--blk-serials', help='Comma separated list of block device serial numbers (or WWNs) to use (requires --lblk).', type=str, default='', dest='blk_serials', required=False) - subcommand.add_argument('--lblk-journal-only', help='With --lblk on a multi-attach cluster whose array plugin creates each node\'s data LUN at add-node: the one selected whole disk is the node\'s journal and nothing else. It is never partitioned, and the node has no local data device; its data device is the array LUN. Needs exactly one disk, named with --blk-names or --blk-serials.', dest='lblk_journal_only', action='store_true') + subcommand.add_argument('--lblk-journal-only', help='With --lblk on a multi-attach cluster whose array plugin creates each node\'s data LUN at add-node and keeps its journal locally: the one selected disk or partition is the node\'s journal and nothing else. It is used whole (a disk is never partitioned), and the node has no local data device; its data device is the array LUN. Needs exactly one disk or partition, named with --blk-names or --blk-serials.', dest='lblk_journal_only', action='store_true') + subcommand.add_argument('--lblk-array-only', help='With --lblk on a multi-attach cluster whose array plugin creates each node\'s LUNs at add-node and keeps its journal on a LUN: the node uses no local device. Its journal LUN and data LUN are created on the array when the node is added. Not with --blk-names, --blk-names-exclude, --blk-serials, --lblk-journal-only or --lblk-journal-device.', dest='lblk_array_only', action='store_true') subcommand.add_argument('--lblk-journal-device', help='With --lblk and whole disks: dedicate each node\'s smallest disk to the journal instead of partitioning it into a journal partition and a data partition (the default). Needs at least two disks per node. Partition selections always carry their journal in a split partition.', dest='lblk_journal_device', action='store_true') subcommand.add_argument('--jm-percent', help='Journal size in percent of the node\'s total selected capacity when the journal is carved from a selected partition or disk (requires --lblk, not with --lblk-journal-device). Default: `3`.', type=int, default=3, dest='jm_percent', required=False) subcommand.add_argument('--force', help='Force format detected or passed nvme pci address to 4K and clean partitions. With --lblk: mark partitioned disks eligible; the partition wipe happens at add-node with --force-format.', dest='force', action='store_true') @@ -587,6 +588,9 @@ def init_cluster__set_array(self, subparser): subcommand.add_argument('--credentials-file', help='A JSON file with the array credentials as an object of strings (for aws-ebs: aws_access_key_id, aws_secret_access_key; omit to use the instance role).', type=str, dest='credentials_file') subcommand.add_argument('--credential-ref', help='Instead of --credentials-file: a secret kept outside simplyblock, as JSON, e.g. `{"kind": "vault-kv", "mount": "secret", "path": "arrays/pure01"}` or `{"kind": "kmip-secret", "uid": "42"}`.', type=str, dest='credential_ref') subcommand.add_argument('--lun-size', help='Size of the LUN created for each node added to the cluster, e.g. `1T`. Default: none.', type=size_type(), default='0', dest='lun_size') + subcommand.add_argument('--journal', help='Where each node\'s journal is kept: `local` (a disk or partition of its host) or `lun` (a LUN of its own, created on the array next to the node\'s data LUN; needs --lun-size). Default: `local`.', type=str, default='local', dest='journal', choices=['local','lun',]) + subcommand.add_argument('--journal-lun-size', help='With --journal lun: size of each node\'s journal LUN, e.g. `64G`. Default: 3%% of --lun-size, at least 2 GiB.', type=size_type(), default='0', dest='journal_lun_size') + subcommand.add_argument('--journal-options', help='With --journal lun: plugin options for the journal LUNs as a JSON object, applied over --options, e.g. `{"pool": "fast-pool"}` to create them in a pool of higher performance or priority.', type=str, dest='journal_options') def init_cluster__get_array(self, subparser): subcommand = self.add_sub_command(subparser, 'get-array', 'Shows the storage array registered for a multi-attach cluster (without its credentials).') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 408c1c9a30..37dfef0ab9 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -279,8 +279,16 @@ def storage_node__configure(self, sub_command, args): if lblk_journal_device: self.parser.error("--lblk-journal-only and --lblk-journal-device exclude each other") if not (blk_names or blk_serials): - self.parser.error("--lblk-journal-only needs the journal disk named with " + self.parser.error("--lblk-journal-only needs the journal disk or partition named with " "--blk-names or --blk-serials") + lblk_array_only = getattr(args, 'lblk_array_only', False) + if lblk_array_only: + if not lblk: + self.parser.error("--lblk-array-only requires --lblk") + if lblk_journal_only or lblk_journal_device or blk_names or blk_names_exclude or blk_serials: + self.parser.error("--lblk-array-only selects no local device: not with --blk-names, " + "--blk-names-exclude, --blk-serials, --lblk-journal-only or " + "--lblk-journal-device") lblk_selection = None if use_lblk: if not lblk: @@ -295,6 +303,7 @@ def storage_node__configure(self, sub_command, args): "serials": [str(x) for x in blk_serials.split(',')] if blk_serials else None, "journal_device": bool(lblk_journal_device), "journal_only": bool(lblk_journal_only), + "array_only": bool(lblk_array_only), } # The core split is decided by the cluster's vcpu-count when the node # is added or restarted; here the default heuristic lays out a usable @@ -887,9 +896,20 @@ def cluster__set_array(self, sub_command, args): raise ValueError(f"--credential-ref is not valid JSON: {e}") if not isinstance(credential_ref, dict): raise ValueError("--credential-ref must be a JSON object") + journal_options = None + if getattr(args, "journal_options", None): + try: + journal_options = json.loads(args.journal_options) + except json.JSONDecodeError as e: + raise ValueError(f"--journal-options is not valid JSON: {e}") + if not isinstance(journal_options, dict): + raise ValueError("--journal-options must be a JSON object") view = array_controller.register_array( args.cluster_id, args.plugin, args.endpoint or "", options, credentials, - int(args.lun_size or 0), credential_ref=credential_ref) + int(args.lun_size or 0), credential_ref=credential_ref, + journal=getattr(args, "journal", None) or "local", + journal_lun_size=int(getattr(args, "journal_lun_size", 0) or 0), + journal_options=journal_options) return _format_result([view], json=getattr(args, "json", False)) def cluster__get_array(self, sub_command, args): diff --git a/simplyblock_core/array/__init__.py b/simplyblock_core/array/__init__.py index b491a9d893..85c97c2237 100644 --- a/simplyblock_core/array/__init__.py +++ b/simplyblock_core/array/__init__.py @@ -339,10 +339,11 @@ def array_credentials(cluster) -> dict: raise ArrayError(f"array credentials could not be read from the KMS: {e}") from e -def plugin_for_cluster(cluster) -> ArrayPlugin: +def plugin_for_cluster(cluster, extra_options: dict | None = None) -> ArrayPlugin: """The connected array plugin a cluster is configured with (``Cluster.array_plugin`` / ``array_endpoint`` / ``array_options``, - credentials from the KMS). + credentials from the KMS). ``extra_options`` update the options for this + instance (the journal LUNs' ``Cluster.array_journal_options``). Raises: ArrayNotConfigured: the cluster names no plugin. @@ -354,7 +355,7 @@ def plugin_for_cluster(cluster) -> ArrayPlugin: return load_plugin( name, getattr(cluster, "array_endpoint", "") or "", - getattr(cluster, "array_options", None) or {}, + dict(getattr(cluster, "array_options", None) or {}, **(extra_options or {})), credentials=array_credentials(cluster), cluster_id=cluster.get_id(), ) diff --git a/simplyblock_core/controllers/array_controller.py b/simplyblock_core/controllers/array_controller.py index 271c367bf1..159d04d262 100644 --- a/simplyblock_core/controllers/array_controller.py +++ b/simplyblock_core/controllers/array_controller.py @@ -16,7 +16,10 @@ - node add: ``provision_node_lun`` creates the node's LUN (``Cluster.array_lun_size``), registers the host, attaches the LUN and waits for it in the node agent's block-device inventory; the add then - continues as a regular lblk add with that device. + continues as a regular lblk add with that device. With the journal on + the array (``Cluster.array_journal`` "lun") it first creates the node's + journal LUN the same way, with the journal options (its pool), and + presents it to that node only. - LVS roles: ``reconcile_lun_attachments`` presents every owner's LUN to its secondary and tertiary (and the node serving it), records the path there in ``node_paths``, and detaches it from a node that lost its role @@ -49,6 +52,16 @@ LUN_GROW_VISIBLE_TIMEOUT_SEC = 180 POLL_SEC = 3 +#: Cluster.array_journal: the journal on a local disk or partition of the +#: node's host, or on a LUN of its own on the array. +JOURNAL_LOCAL = "local" +JOURNAL_LUN = "lun" +JOURNAL_PLACEMENTS = (JOURNAL_LOCAL, JOURNAL_LUN) +#: A journal LUN of no given size: this share of the data LUN, at least +#: JOURNAL_LUN_MIN_BYTES, rounded up to whole GiB. +JOURNAL_LUN_PERCENT = 3 +JOURNAL_LUN_MIN_BYTES = 2 << 30 + _STATUS_REMOVED = "removed" _STATUS_ONLINE = "online" @@ -77,6 +90,21 @@ def provisions_luns(cluster) -> bool: return array_configured(cluster) and int(getattr(cluster, "array_lun_size", 0) or 0) > 0 +def journal_on_lun(cluster) -> bool: + """Node adds also create the node's journal LUN (``Cluster.array_journal`` + "lun" on a cluster that creates the nodes' LUNs).""" + return provisions_luns(cluster) and (getattr(cluster, "array_journal", "") or JOURNAL_LOCAL) == JOURNAL_LUN + + +def journal_lun_size(cluster) -> int: + """Size in bytes of a node's journal LUN.""" + size = int(getattr(cluster, "array_journal_lun_size", 0) or 0) + if size: + return size + size = max(JOURNAL_LUN_MIN_BYTES, math.ceil(int(cluster.array_lun_size or 0) * JOURNAL_LUN_PERCENT / 100)) + return math.ceil(size / (1 << 30)) << 30 + + def _event(cluster_id, message, error=False): try: from simplyblock_core.controllers import events_controller as ec @@ -120,6 +148,11 @@ def array_view(cluster, capabilities=None) -> dict: "endpoint": cluster.array_endpoint, "options": dict(cluster.array_options or {}), "lun_size": int(cluster.array_lun_size or 0), + "journal": getattr(cluster, "array_journal", "") or JOURNAL_LOCAL, + "journal_lun_size": (journal_lun_size(cluster) + if (getattr(cluster, "array_journal", "") or JOURNAL_LOCAL) == JOURNAL_LUN + and int(cluster.array_lun_size or 0) else 0), + "journal_options": dict(getattr(cluster, "array_journal_options", None) or {}), "has_credentials": bool(cluster.array_has_credentials), "credential_ref": array.credential_ref_view(cluster), } @@ -129,7 +162,8 @@ def array_view(cluster, capabilities=None) -> dict: def register_array(cluster_id, plugin, endpoint="", options=None, credentials=None, - lun_size=0, credential_ref=None) -> dict: + lun_size=0, credential_ref=None, journal=JOURNAL_LOCAL, journal_lun_size=0, + journal_options=None) -> dict: """Register (or re-register, e.g. to rotate credentials) the storage array of a multi-attach cluster. @@ -147,10 +181,18 @@ def register_array(cluster_id, plugin, endpoint="", options=None, credentials=No ``{"kind": "kmip-secret", "uid"[, "field"]}``): simplyblock reads it when it connects and never stores it. + ``journal`` places each node's journal: ``"local"`` on a disk or + partition of its host, ``"lun"`` on a LUN of its own that node adds + create (needs ``lun_size``) with ``journal_options`` updating + ``options`` (e.g. ``{"pool": "fast-pool"}``) and ``journal_lun_size`` + bytes (0 = 3% of ``lun_size``, at least 2 GiB). A change applies to + nodes added afterwards. + Returns the array view (``array_view``) with the plugin's capabilities. Raises: - ValueError: not a multi-attach cluster, both or a bad credential + ValueError: not a multi-attach cluster, a bad journal placement, + both or a bad credential source, the credentials cannot be stored or read, the plugin cannot be loaded or connected, its capabilities do not fit the cluster, or LUNs created through another plugin exist. @@ -165,6 +207,17 @@ def register_array(cluster_id, plugin, endpoint="", options=None, credentials=No raise ValueError("an array needs a plugin") if int(lun_size or 0) < 0: raise ValueError("lun_size must not be negative") + journal = (journal or JOURNAL_LOCAL).lower() + if journal not in JOURNAL_PLACEMENTS: + raise ValueError(f"journal must be one of {', '.join(JOURNAL_PLACEMENTS)}, not {journal!r}") + if int(journal_lun_size or 0) < 0: + raise ValueError("journal_lun_size must not be negative") + journal_opts = dict(journal_options or {}) + if journal == JOURNAL_LUN and not int(lun_size or 0): + raise ValueError("a journal LUN is created at node add next to the node's data LUN; " + "it needs lun_size") + if journal != JOURNAL_LUN and (journal_opts or int(journal_lun_size or 0)): + raise ValueError("journal_options and journal_lun_size apply to the journal placement 'lun' only") creds = _wrap_credentials(credentials) if creds and credential_ref: raise ValueError("give the credentials or a reference to them, not both") @@ -199,6 +252,16 @@ def register_array(cluster_id, plugin, endpoint="", options=None, credentials=No raise ValueError(str(e)) from e except Exception as e: # noqa: BLE001 - a vendor plugin raise ValueError(f"array plugin {plugin!r} failed: {e}") from e + if journal_opts: + # The journal LUNs' options are checked by the plugin, as the + # array's are, before anything is saved. + try: + array.load_plugin(plugin, endpoint, dict(opts, **journal_opts), credentials=creds, + cluster_id=cluster_id) + except array.ArrayError as e: + raise ValueError(f"journal options: {e}") from e + except Exception as e: # noqa: BLE001 - a vendor plugin + raise ValueError(f"journal options: array plugin {plugin!r} failed: {e}") from e max_att = int(caps.get("max_attachments") or 0) # Owner, secondary, tertiary: FTT + 1 hosts per LUN. needed = int(getattr(cluster, "max_fault_tolerance", 1) or 1) + 1 @@ -222,6 +285,9 @@ def mutate(fresh): fresh.array_credential_ref = ref fresh.array_has_credentials = bool(ref) fresh.array_lun_size = int(lun_size or 0) + fresh.array_journal = journal + fresh.array_journal_lun_size = int(journal_lun_size or 0) + fresh.array_journal_options = journal_opts db.atomic_update(cluster, mutate) if legacy_secret: @@ -260,7 +326,9 @@ def delete_array(cluster_id, array_id, force=False) -> None: cluster = db.get_cluster_by_id(cluster_id) if not cluster.array_plugin or (cluster.array_id or "") != (array_id or ""): raise KeyError(f"array {array_id} not found") - managed = _managed_devices(db.get_storage_nodes_by_cluster_id(cluster_id)) + nodes = db.get_storage_nodes_by_cluster_id(cluster_id) + managed = [d.array_lun_id for _, d in _managed_devices(nodes)] + managed += [n.array_journal_lun_id for n in nodes if getattr(n, "array_journal_lun_id", "")] if managed and not force: raise ValueError(f"{len(managed)} LUN(s) created through the array exist; " "takeovers would lose their fence") @@ -278,6 +346,9 @@ def mutate(fresh): fresh.array_credential_ref = {} fresh.array_has_credentials = False fresh.array_lun_size = 0 + fresh.array_journal = JOURNAL_LOCAL + fresh.array_journal_lun_size = 0 + fresh.array_journal_options = {} db.atomic_update(cluster, mutate) logger.info("Cluster %s: array %s removed", cluster_id, array_id) @@ -349,8 +420,8 @@ def _set(n): # ---------------------------------------------------------------- node add -def lun_name(hostname, socket) -> str: - return f"simplyblock-{hostname}-s{socket if socket is not None else 0}" +def lun_name(hostname, socket, journal=False) -> str: + return f"simplyblock-{hostname}-s{socket if socket is not None else 0}" + ("-journal" if journal else "") def _find_in_inventory(inventory, wwn): @@ -371,15 +442,20 @@ def _find_in_inventory(inventory, wwn): def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configured, - timeout=LUN_APPEAR_TIMEOUT_SEC, node_ip=""): + timeout=LUN_APPEAR_TIMEOUT_SEC, node_ip="", journal=False): """Create (or find) the LUN of the node being added, present it to the node's host and wait until the host sees it. + With ``journal`` it is the node's journal LUN (``journal_on_lun``): + created with ``Cluster.array_journal_options`` updating the array's + options and ``journal_lun_size`` bytes, and its entry flagged as the + journal. It is presented to this node only, ever. + Returns ``(entry, host_id)``: ``entry`` is an lblk config entry for the LUN (identified by its serial; carries ``lun_wwn`` and ``array_lun_id``) to add to the node's ``lblk_devices``, ``host_id`` the host's id on the - array. A configured entry that already carries an ``array_lun_id`` (a - retried add) is reused. + array. A configured entry of the same kind (data or journal) that already + carries an ``array_lun_id`` (a retried add) is reused. Raises: ArrayLifecycleError: any step failed. @@ -387,24 +463,30 @@ def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configur hostname = node_info.get("hostname", "") socket = node_config.get("socket") try: - plugin = array.plugin_for_cluster(cluster) + plugin = array.plugin_for_cluster( + cluster, dict(getattr(cluster, "array_journal_options", None) or {}) if journal else None) host_id = plugin.ensure_host(hostname, host_initiators( hostname, node_info.get("cloud_instance"), node_info.get("system_id", ""), node_ip, node_info.get("san_initiators"))) - existing = next((e for e in lblk_configured or [] if e.get("array_lun_id")), None) + existing = next((e for e in lblk_configured or [] + if e.get("array_lun_id") and bool(e.get("journal")) == bool(journal)), None) if existing: lun = {"id": existing["array_lun_id"], "wwn": existing.get("lun_wwn", "")} else: - lun = plugin.create_lun(lun_name(hostname, socket), int(cluster.array_lun_size), - {"simplyblock-host": hostname, - "simplyblock-socket": str(socket if socket is not None else 0)}) + tags = {"simplyblock-host": hostname, + "simplyblock-socket": str(socket if socket is not None else 0)} + if journal: + tags["simplyblock-role"] = "journal" + lun = plugin.create_lun(lun_name(hostname, socket, journal), + journal_lun_size(cluster) if journal else int(cluster.array_lun_size), + tags) plugin.attach(lun["id"], host_id) except array.ArrayError as e: raise ArrayLifecycleError(f"array: {e}") from e except Exception as e: # noqa: BLE001 - a vendor plugin raise ArrayLifecycleError(f"array plugin failed: {e}") from e - logger.info("Node %s socket %s: LUN %s (%s) attached as host %s", - hostname, socket, lun["id"], lun["wwn"], host_id) + logger.info("Node %s socket %s: %sLUN %s (%s) attached as host %s", + hostname, socket, "journal " if journal else "", lun["id"], lun["wwn"], host_id) deadline = time.monotonic() + timeout while True: @@ -431,9 +513,17 @@ def provision_node_lun(cluster, snode_api, node_info, node_config, lblk_configur "lun_wwn": lun["wwn"], "array_lun_id": lun["id"], } + if journal: + entry["journal"] = True return entry, host_id +def journal_lun_id(lblk_entries) -> str: + """The array id of the journal LUN among a node's lblk entries ("").""" + return next((e["array_lun_id"] for e in lblk_entries or [] + if e.get("journal") and e.get("array_lun_id")), "") + + def add_lun_to_config(lblk_configured, entry) -> list: """``lblk_configured`` with the LUN entry in place of an older entry for the same LUN (same serial or array id). @@ -759,9 +849,11 @@ def release_node_luns(node) -> None: node_host = getattr(node, "array_host_id", "") or "" for owner, dev in _managed_devices(nodes): if owner.get_id() == node.get_id(): - work.append((dev, None)) + work.append((dev.array_lun_id, None)) elif node_host and dev.serving_node() != node.get_id(): - work.append((dev, node_host)) + work.append((dev.array_lun_id, node_host)) + if getattr(node, "array_journal_lun_id", ""): + work.append((node.array_journal_lun_id, None)) if not work: return plugin = array.plugin_for_cluster(cluster) @@ -770,14 +862,14 @@ def release_node_luns(node) -> None: _event(node.cluster_id, f"LUNs of node {node.get_id()} not released at the array: {e}", error=True) return errors = [] - for dev, host in work: + for lun_id, host in work: try: - hosts = [host] if host else list(plugin.get(dev.array_lun_id).get("hosts") or []) + hosts = [host] if host else list(plugin.get(lun_id).get("hosts") or []) for h in hosts: # The node is gone; its host may not cooperate. - plugin.detach(dev.array_lun_id, h, force=True) + plugin.detach(lun_id, h, force=True) except Exception as e: # noqa: BLE001 - errors.append(f"{dev.array_lun_id}: {e}") + errors.append(f"{lun_id}: {e}") if errors: logger.error("LUNs of node %s not fully released: %s", node.get_id(), errors) _event(node.cluster_id, f"LUNs of node {node.get_id()} not fully released: {errors}", error=True) @@ -796,8 +888,9 @@ def delete_cluster_luns(cluster, force=False) -> None: errors = [] # The LUNs recorded on the cluster's devices, plus whatever the plugin # lists (SAN plugins cannot list: their recorded LUNs are deleted). - recorded = [d.array_lun_id for _, d in - _managed_devices(_db().get_storage_nodes_by_cluster_id(cluster.get_id()))] + nodes = _db().get_storage_nodes_by_cluster_id(cluster.get_id()) + recorded = [d.array_lun_id for _, d in _managed_devices(nodes)] + recorded += [n.array_journal_lun_id for n in nodes if getattr(n, "array_journal_lun_id", "")] luns: list = [] try: plugin = array.plugin_for_cluster(cluster) diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index fcf909b50f..3aea4cd80e 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -262,6 +262,20 @@ def is_topology_owned(self) -> bool: # while an array is registered; 0 = node adds create no LUN (the LUNs are # presented by the operator, as in phase 2/3). array_lun_size: int = 0 + # Where each node's journal is kept when the array plugin creates the + # nodes' LUNs (array_lun_size > 0): + # - "local": a disk or partition of the node's host, named at sn configure + # (--lblk-journal-only); never shared. + # - "lun": a LUN of its own the plugin creates at node add, next to the + # data LUN, presented to that node only (never multi-attached). It is + # created with array_options updated by array_journal_options (e.g. + # {"pool": "fast-pool"}: a pool of higher performance or priority) and + # array_journal_lun_size bytes (0 = 3% of array_lun_size, at least + # 2 GiB). The node's host then needs no local device + # (sn configure --lblk-array-only). + array_journal: str = "local" + array_journal_lun_size: int = 0 + array_journal_options: dict = default_factory(dict) tls: bool = False tls_config: dict = default_factory(dict) is_re_balancing: bool = False diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 42633b91b5..898802adc5 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -238,6 +238,10 @@ class StorageNode(BaseNodeObject): # node's host (ArrayPlugin.ensure_host; the EC2 instance id on EBS). "" # = not known; the fence then names the host by its hostname. array_host_id: str = "" + # Cluster.array_journal "lun": the array's id of this node's journal LUN + # ("" = its journal is local). Released with the node, deleted with the + # cluster, like its data LUN. + array_journal_lun_id: str = "" # Sync-replication site of the node (operator-supplied at add-node). # Mandatory on a cluster with sync_replication, empty everywhere else. site: str = "" diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index f9629cb34b..dd37319b75 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4402,8 +4402,15 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, minimum_sys_memory = int(utils.parse_size(spdk_sys_mem)) else: minimum_sys_memory = int(node_config.get("sys_memory") or 0) - if node_config.get("lblk_devices"): - minimum_sys_memory = max(minimum_sys_memory, utils.node_config_min_sys_memory(node_config)) + if node_config.get("lblk_devices") or node_config.get("lblk_array_only"): + # The LUNs the array plugin creates below count as well. + lun_capacity = 0 + if array_controller.provisions_luns(cluster): + lun_capacity = int(cluster.array_lun_size or 0) + if array_controller.journal_on_lun(cluster): + lun_capacity += array_controller.journal_lun_size(cluster) + minimum_sys_memory = max(minimum_sys_memory, + utils.node_config_min_sys_memory(node_config, lun_capacity)) else: ssd_pcis = node_config.get("ssd_pcis") or [] capacity = utils.nvme_capacity_from_node_info(node_info.get("nvme_devices"), ssd_pcis) @@ -4456,18 +4463,34 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, lblk_mode = cluster.device_mode == constants.DEVICE_MODE_LBLK # Multi-attach cluster with a registered array and a LUN size: the - # node's data LUN is created and presented here (the host's own - # config names the journal device only), then added like any - # configured lblk device. No-op on every other cluster. + # node's data LUN is created and presented here, then added like any + # configured lblk device. Its journal is either a local disk or + # partition the host's own config names (array journal "local"), or + # a journal LUN created here first (array journal "lun"; the host + # config names no device). No-op on every other cluster. array_host_id = "" if lblk_mode and array_controller.provisions_luns(cluster): - if not lblk_configured: + journal_lun = array_controller.journal_on_lun(cluster) + local = [e.get("name") for e in lblk_configured if not e.get("array_lun_id")] + if journal_lun and local: + logger.error( + "This cluster keeps each node's journal on a LUN of its array, but the " + f"node config names local devices {local}; run " + "'sn configure --lblk --lblk-array-only' first.") + return False + if not journal_lun and not local: logger.error( - "This cluster creates each node's LUN on its array, but the node " - "config carries no 'lblk_devices' for the journal; run " - "'sn configure --lblk ' first.") + "This cluster keeps each node's journal on a local disk or partition, but " + "the node config names none; run 'sn configure --lblk --lblk-journal-only " + "--blk-names ' first, or register the array " + "with the journal on a LUN.") return False try: + if journal_lun: + journal_entry, array_host_id = array_controller.provision_node_lun( + cluster, snode_api, node_info, node_config, lblk_configured, + node_ip=node_addr.split(":")[0], journal=True) + lblk_configured = array_controller.add_lun_to_config(lblk_configured, journal_entry) lun_entry, array_host_id = array_controller.provision_node_lun( cluster, snode_api, node_info, node_config, lblk_configured, node_ip=node_addr.split(":")[0]) @@ -4909,6 +4932,7 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, # so restart can re-resolve devices without the host config file. snode.lblk_devices = lblk_configured if lblk_mode else [] snode.array_host_id = array_host_id + snode.array_journal_lun_id = array_controller.journal_lun_id(lblk_configured) if lblk_mode else "" snode.hostname = hostname snode.k8s_node_name = node_info['hostname'] snode.host_nqn = subsystem_nqn diff --git a/simplyblock_core/utils/__init__.py b/simplyblock_core/utils/__init__.py index 61e95535de..6f1fda7df7 100644 --- a/simplyblock_core/utils/__init__.py +++ b/simplyblock_core/utils/__init__.py @@ -2783,18 +2783,17 @@ def previous_lblk_dedicated_journal_serials(config_paths=None) -> set: def lblk_journal_only_layout(lblk_entries, socket=0): - """``sn configure --lblk-journal-only``: the node's one selected whole - disk becomes its journal, flagged, and is never partitioned. The node - then has no local data device: on a multi-attach cluster whose array - plugin creates each node's data LUN at add-node, that LUN is its data - device. Raises ValueError unless exactly one whole disk is selected.""" + """``sn configure --lblk-journal-only``: the node's one selected disk or + partition becomes its journal, flagged, and is used whole (a disk is + never partitioned). The node then has no local data device: on a + multi-attach cluster whose array plugin creates each node's data LUN at + add-node, that LUN is its data device. Raises ValueError unless exactly + one disk or partition is selected.""" if len(lblk_entries) != 1: raise ValueError( - f"--lblk-journal-only needs exactly one disk for the node on socket {socket}; " - f"selected: {sorted(lblk_entries)}") + f"--lblk-journal-only needs exactly one disk or partition for the node on socket " + f"{socket}; selected: {sorted(lblk_entries)}") name, entry = next(iter(lblk_entries.items())) - if entry.get("type") == "part": - raise ValueError(f"--lblk-journal-only needs a whole disk, {name} is a partition") return {name: dict(entry, journal=True)} @@ -2853,7 +2852,11 @@ def lblk_node_journal_layout(lblk_entries, jm_percent=3, journal_device=False, def node_config_device_count(node) -> int: """Number of storage devices a node-config entry carries — ssd_pcis for - nvme mode, lblk_devices for lblk mode.""" + nvme mode, lblk_devices for lblk mode. A node configured with + --lblk-array-only carries none: its array plugin creates its journal LUN + and data LUN at add-node, so it counts those two.""" + if node.get("lblk_array_only"): + return 2 return len(node.get("lblk_devices") or []) or len(node.get("ssd_pcis") or []) @@ -2870,15 +2873,16 @@ def lblk_device_serials(lblk_devices) -> set[str]: return {e["serial"] for e in (lblk_devices or []) if e.get("serial")} -def node_config_min_sys_memory(node) -> int: +def node_config_min_sys_memory(node, extra_capacity=0) -> int: """Minimum system memory for a node-config entry: sys_memory_for_capacity of its total device capacity. lblk entries carry their sizes; nvme entries carry ssd_capacity, measured by generate_configs before it unbound the devices, since nvme-cli no longer sees them afterwards. An entry written - before that field existed goes through nvme-cli.""" + before that field existed goes through nvme-cli. ``extra_capacity`` + (lblk only): the LUNs an array plugin creates for the node at add-node.""" lblk = node.get("lblk_devices") or [] - if lblk: - capacity = sum(int(e.get("size") or 0) for e in lblk) + if lblk or node.get("lblk_array_only"): + capacity = sum(int(e.get("size") or 0) for e in lblk) + int(extra_capacity or 0) elif node.get("ssd_capacity"): capacity = int(node["ssd_capacity"]) else: @@ -3238,6 +3242,8 @@ def regenerate_config(new_config, old_config, force=False): old_config["nodes"][i]["ssd_pcis"] = new_config["nodes"][i]["ssd_pcis"] if new_config["nodes"][i].get("lblk_devices") is not None: old_config["nodes"][i]["lblk_devices"] = new_config["nodes"][i]["lblk_devices"] + if "lblk_array_only" in new_config["nodes"][i]: + old_config["nodes"][i]["lblk_array_only"] = new_config["nodes"][i]["lblk_array_only"] old_config["nodes"][i]["nic_ports"] = new_config["nodes"][i]["nic_ports"] for nic in old_config["nodes"][i]["nic_ports"]: if nic not in all_nics: @@ -3299,7 +3305,14 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a nics = detect_nics() lblk_mode = lblk_selection is not None lblk_entries: dict = {} - if lblk_mode: + array_only = bool(lblk_mode and lblk_selection.get("array_only")) + if array_only: + # sn configure --lblk-array-only: no local device at all. The array + # plugin creates the node's journal LUN and data LUN at add-node. + lblk_units: dict = {} + lblk_keep_serials: set = set() + nvmes: dict = {} + elif lblk_mode: # lblk cluster mode: eligible Linux block devices instead of NVMe # PCIe controllers. No driver unbind, no formatting here (--force # only marks partitioned disks eligible; the wipe happens at @@ -3470,7 +3483,10 @@ def generate_configs(max_lvol, max_prov, sockets_to_use, nodes_per_socket, pci_a node_info["number_of_distribs"] = number_of_distribs nvme_neg1_list = all_nvmes_neg1_per_node[node_index] - if lblk_mode: + if array_only: + node_info["lblk_devices"] = [] + node_info["lblk_array_only"] = True + elif lblk_mode: node_entries = {} for unit in list(nvme_neg1_list) + list(nvme_per_core_group[idx]): for dev_name in lblk_units[unit]: @@ -3910,7 +3926,14 @@ def validate_node_config(node): # Exactly one device source: PCIe SSDs (nvme mode) or Linux block # devices (lblk mode). Both empty, or both populated, is a broken config. lblk_devices = node.get("lblk_devices") or [] - if bool(node["ssd_pcis"]) == bool(lblk_devices): + if node.get("lblk_array_only"): + # sn configure --lblk-array-only: no local device; the array plugin + # creates the node's journal and data LUNs at add-node. + if node["ssd_pcis"] or lblk_devices: + logger.error(f"A node configured with --lblk-array-only carries no devices, " + f"node {node.get('socket')} does") + return False + elif bool(node["ssd_pcis"]) == bool(lblk_devices): logger.error( f"Node config must carry exactly one non-empty device list of " f"'ssd_pcis' / 'lblk_devices' in node: {node.get('socket')}") diff --git a/simplyblock_web/api/v2/cluster/array.py b/simplyblock_web/api/v2/cluster/array.py index d7bcbb0fe5..69337a24ac 100644 --- a/simplyblock_web/api/v2/cluster/array.py +++ b/simplyblock_web/api/v2/cluster/array.py @@ -7,7 +7,7 @@ record. See simplyblock_core.controllers.array_controller. """ -from typing import Annotated, Any +from typing import Annotated, Any, Literal from fastapi import APIRouter, HTTPException, Response from pydantic import BaseModel, Field, SecretStr @@ -32,6 +32,14 @@ class ArrayParams(BaseModel): '{"kind": "vault-kv", "mount": ..., "path": ..., "version": ...} or ' '{"kind": "kmip-secret", "uid": ..., "field": ...}'))] = Field(default_factory=dict) lun_size: Annotated[util.Size, Field(description="Size of the LUN created for each added node; 0 = none")] = 0 + journal: Annotated[Literal["local", "lun"], Field(description=( + "Where each node's journal is kept: local (a disk or partition of its host) or lun " + "(a LUN of its own, created next to its data LUN; needs lun_size)"))] = "local" + journal_lun_size: Annotated[util.Size, Field(description=( + "journal lun: size of each node's journal LUN; 0 = 3% of lun_size, at least 2 GiB"))] = 0 + journal_options: Annotated[dict[str, Any], Field(description=( + 'journal lun: plugin options for the journal LUNs, applied over options, e.g. {"pool": "fast-pool"}' + ))] = Field(default_factory=dict) class ArrayDTO(BaseModel): @@ -40,6 +48,9 @@ class ArrayDTO(BaseModel): endpoint: str options: dict[str, Any] lun_size: int + journal: str = "local" + journal_lun_size: int = 0 + journal_options: dict[str, Any] = Field(default_factory=dict) has_credentials: bool credential_ref: dict[str, Any] = Field(default_factory=dict) capabilities: dict[str, Any] | None = None @@ -54,7 +65,8 @@ def register(cluster: Cluster, parameters: ArrayParams) -> ArrayDTO: view = array_controller.register_array( cluster.get_id(), parameters.plugin, parameters.endpoint, parameters.options, parameters.credentials, parameters.lun_size, - credential_ref=parameters.credential_ref or None) + credential_ref=parameters.credential_ref or None, journal=parameters.journal, + journal_lun_size=parameters.journal_lun_size, journal_options=parameters.journal_options or None) except ValueError as e: raise HTTPException(400, str(e)) return ArrayDTO(**view) diff --git a/simplyblock_web/node_configure.py b/simplyblock_web/node_configure.py index 5ff8bea869..0e20366062 100755 --- a/simplyblock_web/node_configure.py +++ b/simplyblock_web/node_configure.py @@ -226,6 +226,12 @@ def parse_arguments() -> argparse.Namespace: action='store_true', dest='lblk_journal_only', ) + parser.add_argument( + '--lblk-array-only', + help="With --lblk on a multi-attach cluster whose array plugin creates each node's LUNs at add-node and keeps its journal on a LUN: the node uses no local device. Its journal LUN and data LUN are created on the array when the node is added. Not with --blk-names, --blk-names-exclude, --blk-serials, --lblk-journal-only or --lblk-journal-device.", + action='store_true', + dest='lblk_array_only', + ) parser.add_argument( '--lblk-journal-device', help='With --lblk and whole disks: dedicate each node\'s smallest disk to the ' @@ -304,8 +310,17 @@ def validate_arguments(args: argparse.Namespace) -> None: None, "--lblk-journal-only and --lblk-journal-device exclude each other") if not (args.blk_names or args.blk_serials): raise argparse.ArgumentError( - None, "--lblk-journal-only needs the journal disk named with " + None, "--lblk-journal-only needs the journal disk or partition named with " "--blk-names or --blk-serials") + if getattr(args, 'lblk_array_only', False): + if not lblk: + raise argparse.ArgumentError(None, "--lblk-array-only requires --lblk") + if (getattr(args, 'lblk_journal_only', False) or getattr(args, 'lblk_journal_device', False) + or blk_names or blk_names_exclude or blk_serials): + raise argparse.ArgumentError( + None, "--lblk-array-only selects no local device: not with --blk-names, " + "--blk-names-exclude, --blk-serials, --lblk-journal-only or " + "--lblk-journal-device") if use_lblk and (args.pci_allowed or args.pci_blocked or getattr(args, 'device_model', '') or getattr(args, 'size_range', '') @@ -384,6 +399,7 @@ def main() -> None: "serials": [x.strip() for x in args.blk_serials.split(',') if x.strip()] or None, "journal_device": bool(getattr(args, 'lblk_journal_device', False)), "journal_only": bool(getattr(args, 'lblk_journal_only', False)), + "array_only": bool(getattr(args, 'lblk_array_only', False)), # Read before the discard below: the disks of an existing node # with a dedicated journal device are never partitioned. "keep_serials": sorted(utils.previous_lblk_dedicated_journal_serials()), diff --git a/tests/unit/test_array_journal.py b/tests/unit/test_array_journal.py new file mode 100644 index 0000000000..74c1c80cbe --- /dev/null +++ b/tests/unit/test_array_journal.py @@ -0,0 +1,297 @@ +"""Where a node's journal is kept on a multi-attach cluster whose array plugin +creates the nodes' LUNs (Cluster.array_journal): + + - "lun": node add creates a journal LUN next to the data LUN, with the + journal options (a faster pool) over the array's options and its own + size, presented to that node only; the host config names no device + (sn configure --lblk-array-only). The LUN is recorded on the node, + detached with it and deleted with the cluster. + - "local": the host config names one local disk or partition + (sn configure --lblk-journal-only), used whole. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from simplyblock_core import utils +from simplyblock_core.array.fake import FakeArrayPlugin +from simplyblock_core.controllers import array_controller +from tests.unit import test_lblk_journal_disk_split as journal_split +from tests.unit.test_array_lifecycle import ( + CID, + GIB, + PLUGIN, + FakeDB, + _Case, + _cluster, + _lun_device, + _node, +) + + +def _journal_cluster(journal="lun", lun_size=100 * GIB, journal_lun_size=0, journal_options=None): + cl = _cluster(lun_size=lun_size) + cl.array_journal = journal + cl.array_journal_lun_size = journal_lun_size + cl.array_journal_options = dict(journal_options or {}) + return cl + + +class TestRegistration(_Case): + + def test_journal_lun_with_its_own_pool(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + view = array_controller.register_array( + CID, PLUGIN, options={"pool": "capacity"}, lun_size=100 * GIB, journal="lun", + journal_lun_size=8 * GIB, journal_options={"pool": "fast"}) + self.assertEqual((db.cluster.array_journal, db.cluster.array_journal_lun_size, + db.cluster.array_journal_options), ("lun", 8 * GIB, {"pool": "fast"})) + self.assertEqual((view["journal"], view["journal_lun_size"], view["journal_options"]), + ("lun", 8 * GIB, {"pool": "fast"})) + # The journal options were checked by the plugin, over the array's. + self.assertIn({"pool": "fast"}, [c[2] for c in FakeArrayPlugin.connected]) + # The array's own options are unchanged. + self.assertEqual(db.cluster.array_options, {"pool": "capacity"}) + + def test_default_is_local(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + view = array_controller.register_array(CID, PLUGIN, lun_size=GIB) + self.assertEqual((db.cluster.array_journal, view["journal"], view["journal_lun_size"]), + ("local", "local", 0)) + + def test_refusals(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + for kw, msg in (({"journal": "nvram"}, "journal must be one of"), + ({"journal": "lun"}, "needs lun_size"), + ({"lun_size": GIB, "journal_options": {"pool": "f"}}, "'lun' only"), + ({"lun_size": GIB, "journal_lun_size": GIB}, "'lun' only"), + ({"lun_size": GIB, "journal": "lun", "journal_lun_size": -1}, "negative")): + with self.subTest(kw=kw), self.assertRaisesRegex(ValueError, msg): + array_controller.register_array(CID, PLUGIN, **kw) + self.assertEqual(db.cluster.array_plugin, "") + + def test_bad_journal_options_save_nothing(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + real_connect = FakeArrayPlugin.connect + + def connect(self, endpoint, credentials, options): + if options.get("pool") == "missing": + raise array_controller.array.ArrayError("no pool 'missing'") + return real_connect(self, endpoint, credentials, options) + with patch.object(FakeArrayPlugin, "connect", connect), \ + self.assertRaisesRegex(ValueError, "journal options: .*no pool"): + array_controller.register_array(CID, PLUGIN, lun_size=GIB, journal="lun", + journal_options={"pool": "missing"}) + self.assertEqual(db.cluster.array_plugin, "") + + def test_delete_resets_and_counts_journal_luns(self): + a = _node("a") + db = self.use_db(FakeDB(_cluster(plugin=""), [a])) + array_controller.register_array(CID, PLUGIN, lun_size=GIB, journal="lun", + journal_options={"pool": "fast"}) + a.array_journal_lun_id = "lun-j" + with self.assertRaisesRegex(ValueError, "1 LUN"): + array_controller.delete_array(CID, db.cluster.array_id) + array_controller.delete_array(CID, db.cluster.array_id, force=True) + self.assertEqual((db.cluster.array_journal, db.cluster.array_journal_options), ("local", {})) + + +class TestSize(unittest.TestCase): + + def test_default_is_3_percent_at_least_2_gib_whole_gib(self): + self.assertEqual(array_controller.journal_lun_size(_journal_cluster(lun_size=1024 * GIB)), + 31 * GIB) # 30.72 GiB rounded up + self.assertEqual(array_controller.journal_lun_size(_journal_cluster(lun_size=10 * GIB)), 2 * GIB) + self.assertEqual(array_controller.journal_lun_size( + _journal_cluster(journal_lun_size=5 * GIB + 1)), 5 * GIB + 1) + + def test_gate(self): + self.assertTrue(array_controller.journal_on_lun(_journal_cluster())) + self.assertFalse(array_controller.journal_on_lun(_journal_cluster(journal="local"))) + self.assertFalse(array_controller.journal_on_lun(_journal_cluster(lun_size=0))) + cl = _journal_cluster() + cl.array_journal = "" # a record from before the field + self.assertFalse(array_controller.journal_on_lun(cl)) + + +class TestProvisioning(_Case): + + def _api(self): + """Every LUN the fake array holds is a disk of the host.""" + api = MagicMock() + api.scan_new_luns.return_value = ({}, None) + + def inv(): + return [{"type": "disk", "name": f"sd{i}", "serial": lun["wwn"], "wwn": "", + "size": lun["size"], "by_id_path": f"/dev/disk/by-id/{lun['wwn']}", "numa_node": 0} + for i, lun in enumerate(FakeArrayPlugin.luns.values())], None + api.get_blockdevices.side_effect = inv + return api + + def test_journal_lun_then_data_lun(self): + cl = _journal_cluster(journal_options={"pool": "fast"}, journal_lun_size=4 * GIB) + cl.array_options = {"pool": "capacity", "thin": True} + api, info = self._api(), {"hostname": "w1"} + jentry, host = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, [], journal=True) + cfg = array_controller.add_lun_to_config([], jentry) + dentry, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, cfg) + cfg = array_controller.add_lun_to_config(cfg, dentry) + + jlun = FakeArrayPlugin.luns[jentry["array_lun_id"]] + dlun = FakeArrayPlugin.luns[dentry["array_lun_id"]] + self.assertEqual((jlun["name"], jlun["size"], jlun["hosts"]), ("simplyblock-w1-s0-journal", 4 * GIB, [host])) + self.assertEqual(jlun["tags"]["simplyblock-role"], "journal") + self.assertEqual((dlun["name"], dlun["size"]), ("simplyblock-w1-s0", 100 * GIB)) + # The journal LUN was created through a plugin with the journal pool, + # the data LUN through one with the array's own options. + pools = [c[2].get("pool") for c in FakeArrayPlugin.connected] + self.assertEqual(pools, ["fast", "capacity"]) + self.assertEqual(FakeArrayPlugin.connected[0][2]["thin"], True) + # The journal entry is flagged, the data entry is not, and both stay. + self.assertEqual([bool(e.get("journal")) for e in cfg], [True, False]) + self.assertEqual(array_controller.journal_lun_id(cfg), jentry["array_lun_id"]) + + def test_retried_add_reuses_each_kind(self): + cl = _journal_cluster() + api, info = self._api(), {"hostname": "w1"} + jentry, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, [], journal=True) + dentry, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, [jentry]) + cfg = [jentry, dentry] + j2, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, cfg, journal=True) + d2, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, cfg) + self.assertEqual((j2["array_lun_id"], d2["array_lun_id"]), + (jentry["array_lun_id"], dentry["array_lun_id"])) + self.assertEqual(len(FakeArrayPlugin.luns), 2) + + def test_journal_lun_is_not_multi_attached(self): + from simplyblock_core import multi_attach + from simplyblock_core.models.nvme_device import NVMeDevice + cl = _journal_cluster() + api, info = self._api(), {"hostname": "w1"} + jentry, _ = array_controller.provision_node_lun(cl, api, info, {"socket": 0}, [], journal=True) + dev = NVMeDevice() + dev.serial_number = jentry["serial"] + multi_attach.mark_lun_devices(cl, _node("a"), [dev], [jentry]) + self.assertFalse(dev.multi_attach) + self.assertEqual(dev.array_lun_id, "") + + +class TestReleaseAndDelete(_Case): + + def test_node_release_detaches_its_journal_lun(self): + a = _node("a") + db = self.use_db(FakeDB(_journal_cluster(), [a])) + p = self.plugin(db.cluster) + data, journal = p.create_lun("d", GIB, {}), p.create_lun("j", GIB, {}) + for lun in (data, journal): + p.attach(lun["id"], "host-a") + _lun_device(a, data["id"], data["wwn"]) + a.array_journal_lun_id = journal["id"] + array_controller.release_node_luns(a) + self.assertEqual((p.get(data["id"])["hosts"], p.get(journal["id"])["hosts"]), ([], [])) + self.assertEqual(len(FakeArrayPlugin.luns), 2) # released, not deleted + + def test_cluster_delete_deletes_recorded_journal_luns(self): + a = _node("a") + cl = _journal_cluster() + self.use_db(FakeDB(cl, [a])) + p = self.plugin(cl) + journal = p.create_lun("j", GIB, {}) + a.array_journal_lun_id = journal["id"] + with patch.object(FakeArrayPlugin, "list_luns", side_effect=NotImplementedError): + array_controller.delete_cluster_luns(cl) + self.assertEqual(FakeArrayPlugin.luns, {}) + + +class TestHostConfig(unittest.TestCase): + """sn configure: --lblk-array-only (no local device) and a local journal + on a partition (--lblk-journal-only).""" + + def test_journal_only_accepts_a_partition(self): + out = utils.lblk_journal_only_layout( + {"nvme0n1p3": {"name": "nvme0n1p3", "serial": "P", "size": GIB, "type": "part"}}) + self.assertTrue(out["nvme0n1p3"]["journal"]) + + def test_array_only_node_counts_its_two_luns(self): + self.assertEqual(utils.node_config_device_count({"lblk_devices": [], "lblk_array_only": True}), 2) + self.assertEqual(utils.node_config_device_count({"lblk_devices": [], "ssd_pcis": []}), 0) + + def test_array_only_sys_memory_counts_the_luns(self): + node = {"lblk_devices": [], "lblk_array_only": True} + self.assertEqual(utils.node_config_min_sys_memory(node, 10 * 1024 * GIB), + int(utils.sys_memory_for_capacity(10 * 1024 * GIB))) + + def _node(self, **kw): + node = {"socket": 0, "cpu_mask": "0x2", "isolated": [1], "l-cores": "1@1", "ssd_pcis": [], + "lblk_devices": [], "distribution": {}} + node.update(kw) + return node + + def test_validate_array_only(self): + fields = {f: [] for f in ("app_thread_core", "jm_cpu_core", "poller_cpu_cores", "alceml_cpu_cores", + "distrib_cpu_cores", "jc_singleton_core")} + with patch.object(utils, "validate_node_config", wraps=utils.validate_node_config): + base = self._node(distribution=fields) + for k in ("max_lvol", "max_size", "huge_page_memory", "number_of_distribs", "nic_ports", + "number_of_alcemls", "small_pool_count", "large_pool_count", "sys_memory"): + base.setdefault(k, 1) + self.assertFalse(utils.validate_node_config(dict(base))) + self.assertTrue(utils.validate_node_config(dict(base, lblk_array_only=True))) + self.assertFalse(utils.validate_node_config(dict( + base, lblk_array_only=True, + lblk_devices=[{"name": "sdb", "serial": "S", "size": GIB}]))) + + +class TestGenerateArrayOnly(unittest.TestCase): + # The harness only; its tests stay in their module. + _run = journal_split.TestGenerateConfigsPerNode._run + + def test_array_only_node_has_no_device(self): + config, calls = self._run({}, nodes_per_socket=2, selection={"array_only": True}) + self.assertEqual(calls, []) + self.assertEqual([(n["lblk_devices"], n["lblk_array_only"]) for n in config["nodes"]], + [([], True), ([], True)]) + + +class TestConfigureArgs(unittest.TestCase): + + def _args(self, **kw): + import argparse + base = {"max_lvol": "10", "max_prov": "", "pci_allowed": "", "pci_blocked": "", "lblk": True, + "blk_names": "", "blk_names_exclude": "", "blk_serials": "", "lblk_journal_only": False, + "lblk_journal_device": False, "lblk_array_only": True, "device_model": "", "size_range": "", + "nvme_names": "", "upgrade": False} + base.update(kw) + return argparse.Namespace(**base) + + def test_node_configure_validation(self): + import argparse + + from simplyblock_web import node_configure + node_configure.validate_arguments(self._args()) + for kw in ({"lblk": False}, {"blk_names": "sdb"}, {"lblk_journal_only": True}, + {"lblk_journal_device": True}): + with self.subTest(kw=kw), self.assertRaises(argparse.ArgumentError): + node_configure.validate_arguments(self._args(**kw)) + + def test_cli_has_the_flags(self): + from simplyblock_cli import cli + with open(cli.__file__, encoding="utf-8") as f: + text = f.read() + for flag in ("'--lblk-array-only'", "'--journal'", "'--journal-lun-size'", "'--journal-options'"): + self.assertIn(flag, text) + + def test_cli_set_array_passes_the_journal(self): + from simplyblock_cli.clibase import CLIWrapperBase + args = MagicMock(cluster_id=CID, plugin="p", endpoint="", options=None, credentials_file=None, + credential_ref=None, lun_size=GIB, journal="lun", journal_lun_size=8 * GIB, + journal_options='{"pool": "fast"}', json=True) + with patch.object(array_controller, "register_array", return_value={"id": "arr"}) as reg: + CLIWrapperBase.cluster__set_array(MagicMock(), "set-array", args) + kw = reg.call_args.kwargs + self.assertEqual((kw["journal"], kw["journal_lun_size"], kw["journal_options"]), + ("lun", 8 * GIB, {"pool": "fast"})) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_array_lifecycle.py b/tests/unit/test_array_lifecycle.py index 42d0e11497..c539fdae6f 100644 --- a/tests/unit/test_array_lifecycle.py +++ b/tests/unit/test_array_lifecycle.py @@ -1166,7 +1166,8 @@ def test_set_array_reads_and_wraps_credentials(self): json.dump({"aws_access_key_id": "AK", "aws_secret_access_key": "SK"}, f) self.addCleanup(os.unlink, f.name) args = MagicMock(cluster_id=CID, plugin="aws-ebs", endpoint="", options='{"az": "z"}', - credentials_file=f.name, credential_ref=None, lun_size=GIB, json=True) + credentials_file=f.name, credential_ref=None, lun_size=GIB, journal="local", + journal_lun_size=0, journal_options=None, json=True) with patch.object(array_controller, "register_array", return_value={"id": "arr"}) as reg: CLIWrapperBase.cluster__set_array(MagicMock(), "set-array", args) a = reg.call_args.args diff --git a/tests/unit/test_lblk_journal_only.py b/tests/unit/test_lblk_journal_only.py index ccbf16991b..eb9ec33ef8 100644 --- a/tests/unit/test_lblk_journal_only.py +++ b/tests/unit/test_lblk_journal_only.py @@ -24,16 +24,16 @@ def test_input_is_not_modified(self): self.assertNotIn("journal", entries["sdb"]) def test_two_disks_refused(self): - with self.assertRaisesRegex(ValueError, "exactly one disk"): + with self.assertRaisesRegex(ValueError, "exactly one disk or partition"): utils.lblk_journal_only_layout({"sdb": _disk(), "sdc": _disk("sdc")}, socket=1) def test_no_disk_refused(self): - with self.assertRaisesRegex(ValueError, "exactly one disk"): + with self.assertRaisesRegex(ValueError, "exactly one disk or partition"): utils.lblk_journal_only_layout({}) - def test_partition_refused(self): - with self.assertRaisesRegex(ValueError, "whole disk"): - utils.lblk_journal_only_layout({"sdb1": _disk("sdb1", type="part")}) + def test_partition_is_a_journal_too(self): + out = utils.lblk_journal_only_layout({"sdb1": _disk("sdb1", type="part")}) + self.assertTrue(out["sdb1"]["journal"]) class TestConfigureArgs(unittest.TestCase): From f2a518c2c6f3564b9c216011c13976911fb4c040 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 20:56:20 +0100 Subject: [PATCH 42/50] feat(array): grow every LUN to a size (lun_size), repeatable after a failure grow-devices takes lun_size (API ?lun_size=, CLI --lun-size) besides lun_factor: every array-created data LUN is grown to that many bytes, a LUN already that large is left alone, so a growth that failed half-way is simply repeated without growing a LUN twice (a factor would compound). Once every LUN has the size, Cluster.array_lun_size becomes it, so nodes added later match. This is what the operator drives from spec.multiAttach.array.lunSize. Co-Authored-By: Claude Opus 5.5 --- simplyblock_cli/cli-reference.yaml | 8 +++ simplyblock_cli/cli.py | 1 + simplyblock_cli/clibase.py | 8 ++- .../controllers/array_controller.py | 48 +++++++++++++----- .../controllers/device_controller.py | 12 +++-- simplyblock_web/api/v2/cluster/__init__.py | 16 +++++- tests/unit/test_array_lifecycle.py | 49 +++++++++++++++++++ 7 files changed, 122 insertions(+), 20 deletions(-) diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 8d99bf5d3b..d99c2b1c36 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -1612,6 +1612,14 @@ commands: this factor (greater than 1, e.g. 1.5) at the array and wait until every node sees the new size. dest: lun_factor type: str + - name: "--lun-size" + help: > + Multi-attach cluster with a registered array only, instead of --lun-factor: first grow every LUN the + array plugin created to this size (e.g. `6T`); a LUN already that large is left alone, so the command + can be repeated after a failure. Nodes added later get LUNs of this size. + dest: lun_size + type: size + default: "0" - name: set-array help: "Registers the storage array of a multi-attach cluster (or updates it, e.g. to rotate credentials)." usage: > diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 1f05e9705b..2989a04be7 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -578,6 +578,7 @@ def init_cluster__grow_devices(self, subparser): subcommand = self.add_sub_command(subparser, 'grow-devices', 'Grows all Linux block devices (lblk) of the cluster online by the same factor.') subcommand.add_argument('cluster_id', help='The cluster id.', type=str).completer = self._completer_get_cluster_list subcommand.add_argument('--lun-factor', help='Multi-attach cluster with a registered array only: first grow every LUN the array plugin created by this factor (greater than 1, e.g. 1.5) at the array and wait until every node sees the new size.', type=str, dest='lun_factor') + subcommand.add_argument('--lun-size', help='Multi-attach cluster with a registered array only, instead of --lun-factor: first grow every LUN the array plugin created to this size (e.g. `6T`); a LUN already that large is left alone, so the command can be repeated after a failure. Nodes added later get LUNs of this size.', type=size_type(), default='0', dest='lun_size') def init_cluster__set_array(self, subparser): subcommand = self.add_sub_command(subparser, 'set-array', 'Registers the storage array of a multi-attach cluster (or updates it, e.g. to rotate credentials).') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 37dfef0ab9..769a139007 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -847,7 +847,13 @@ def cluster__grow_devices(self, sub_command, args): except ValueError: print(f"--lun-factor must be a number, not {lun_factor!r}") return False - extra = {"lun_factor": lun_factor} if lun_factor is not None else {} + extra: dict = {"lun_factor": lun_factor} if lun_factor is not None else {} + lun_size = int(getattr(args, 'lun_size', 0) or 0) + if lun_size: + if lun_factor is not None: + print("Give --lun-factor or --lun-size, not both") + return False + extra = {"lun_size": lun_size} try: result = device_controller.grow_cluster_devices(args.cluster_id, **extra) except ValueError as e: diff --git a/simplyblock_core/controllers/array_controller.py b/simplyblock_core/controllers/array_controller.py index 159d04d262..c384a2caa8 100644 --- a/simplyblock_core/controllers/array_controller.py +++ b/simplyblock_core/controllers/array_controller.py @@ -757,24 +757,41 @@ def reconcile_after_role_change(cluster_id) -> dict: # ------------------------------------------------------------------ growth -def grow_luns(cluster_id, factor, visible_timeout=LUN_GROW_VISIBLE_TIMEOUT_SEC) -> dict: - """Grow every array-created LUN of the cluster by ``factor`` (> 1) at the - array, then wait until every ONLINE node the LUN is presented to sees the - new size (agent rescan; dm-multipath maps are resized too). +def grow_luns(cluster_id, factor=None, visible_timeout=LUN_GROW_VISIBLE_TIMEOUT_SEC, size=None) -> dict: + """Grow every array-created data LUN of the cluster at the array, then + wait until every ONLINE node the LUN is presented to sees the new size + (agent rescan; dm-multipath maps are resized too). + + Either by ``factor`` (> 1, relative to each LUN's size now), or to + ``size`` bytes: a LUN already that large is left alone, so a growth to a + size can be repeated after a failure without growing any LUN twice, and + the cluster's LUN size for nodes added later (``Cluster.array_lun_size``) + becomes ``size`` once every LUN has it. Returns ``{"luns": {device_id: {"lun_id", "old_size", "new_size"}}, "errors": {device_id: msg}}``. The device growth itself (alceml, weights) is the regular grow-devices path that follows. Raises: - ValueError: not an array cluster, or a bad factor. + ValueError: not an array cluster, a bad factor or size, or both or + neither given. """ - try: - factor = float(factor) - except (TypeError, ValueError) as e: - raise ValueError("the growth factor must be a number") from e - if not math.isfinite(factor) or factor <= 1.0: - raise ValueError("the growth factor must be greater than 1") + if (factor is None) == (size is None): + raise ValueError("give a growth factor or a LUN size, not both") + if size is not None: + try: + size = int(size) + except (TypeError, ValueError) as e: + raise ValueError("the LUN size must be a number of bytes") from e + if size <= 0: + raise ValueError("the LUN size must be positive") + else: + try: + factor = float(factor) + except (TypeError, ValueError) as e: + raise ValueError("the growth factor must be a number") from e + if not math.isfinite(factor) or factor <= 1.0: + raise ValueError("the growth factor must be greater than 1") db = _db() cluster = db.get_cluster_by_id(cluster_id) if not array_configured(cluster): @@ -792,8 +809,9 @@ def grow_luns(cluster_id, factor, visible_timeout=LUN_GROW_VISIBLE_TIMEOUT_SEC) did = dev.get_id() try: old = int(plugin.get(dev.array_lun_id).get("size") or 0) or int(dev.size) - target = math.ceil(old * factor) - plugin.resize(dev.array_lun_id, target) + target = max(old, size) if size is not None else math.ceil(old * factor) + if target > old: + plugin.resize(dev.array_lun_id, target) new = int(plugin.get(dev.array_lun_id).get("size") or target) out["luns"][did] = {"lun_id": dev.array_lun_id, "old_size": old, "new_size": new} except Exception as e: # noqa: BLE001 @@ -802,6 +820,10 @@ def grow_luns(cluster_id, factor, visible_timeout=LUN_GROW_VISIBLE_TIMEOUT_SEC) err = _wait_visible(dev, nodes_by_id, new, visible_timeout) if err: out["errors"][did] = err + if size is not None and not out["errors"] and size > int(cluster.array_lun_size or 0): + def _set(fresh): + fresh.array_lun_size = max(int(fresh.array_lun_size or 0), size) + db.atomic_update(cluster, _set) return out diff --git a/simplyblock_core/controllers/device_controller.py b/simplyblock_core/controllers/device_controller.py index 20abaa60b5..28b2044541 100644 --- a/simplyblock_core/controllers/device_controller.py +++ b/simplyblock_core/controllers/device_controller.py @@ -2388,7 +2388,7 @@ def _device_report(dev, old_size): "placement_size": dev.weight_size()} -def grow_cluster_devices(cluster_id, lun_factor=None): +def grow_cluster_devices(cluster_id, lun_factor=None, lun_size=None): """Grow every data device of an lblk cluster by the same factor (see _grow_cluster_devices). @@ -2398,8 +2398,12 @@ def grow_cluster_devices(cluster_id, lun_factor=None): (array_controller.grow_luns). If any LUN fails, nothing is grown on the cluster; the uniform growth below would refuse anyway, since growth takes the smallest factor of all block devices. The result then also carries - ``luns``.""" - if lun_factor is None: + ``luns``. + + ``lun_size`` instead of ``lun_factor``: grow every LUN to that many bytes + (a LUN already that large is left alone, so a failed growth is simply + repeated) and make it the LUN size of nodes added later.""" + if lun_factor is None and lun_size is None: return _grow_cluster_devices(cluster_id) from simplyblock_core.controllers import array_controller # Check before the array is touched: a refusal after the LUNs grew would @@ -2415,7 +2419,7 @@ def grow_cluster_devices(cluster_id, lun_factor=None): "the array") result["luns"] = {} return result - lun_report = array_controller.grow_luns(cluster_id, lun_factor) + lun_report = array_controller.grow_luns(cluster_id, lun_factor, size=lun_size) if lun_report["errors"]: result = _growth_result(False, 1.0, [], dict(lun_report["errors"]), "Not every LUN could be grown at the array; nothing was grown " diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index 736f5a4a4d..b8b06893d7 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -436,7 +436,8 @@ class DeviceGrowthDTO(BaseModel): @instance_api.post('/grow-devices', name='clusters:grow_devices', response_model=DeviceGrowthDTO) def grow_devices(cluster: Cluster, - lun_factor: Annotated[float | None, Query(gt=1)] = None) -> DeviceGrowthDTO: + lun_factor: Annotated[float | None, Query(gt=1)] = None, + lun_size: Annotated[int | None, Query(gt=0)] = None) -> DeviceGrowthDTO: """Grow all lblk data devices of the cluster online by the same factor. Every block device is rescanned (for example, after its LUN was grown on @@ -446,10 +447,21 @@ def grow_devices(cluster: Cluster, (`grown` false, `reason`). Device weights and data placement never change. A block device smaller than its device is refused, listed in `shrunk` and `errors`, and raises the device_shrunk alert. + + On a cluster whose array plugin creates its LUNs, every LUN is grown at + the array first: by `lun_factor`, or to `lun_size` bytes (a LUN already + that large is left alone, so the call can be repeated after a failure; + nodes added later get LUNs of that size). """ if cluster.device_mode != constants.DEVICE_MODE_LBLK: raise HTTPException(409, 'Online device growth needs a cluster in lblk device mode') - extra = {"lun_factor": lun_factor} if lun_factor is not None else {} + if lun_factor is not None and lun_size is not None: + raise HTTPException(400, 'Give lun_factor or lun_size, not both') + extra: dict = {} + if lun_factor is not None: + extra["lun_factor"] = lun_factor + if lun_size is not None: + extra["lun_size"] = lun_size try: result = device_controller.grow_cluster_devices(cluster.get_id(), **extra) except ValueError as e: diff --git a/tests/unit/test_array_lifecycle.py b/tests/unit/test_array_lifecycle.py index c539fdae6f..24b364cd2e 100644 --- a/tests/unit/test_array_lifecycle.py +++ b/tests/unit/test_array_lifecycle.py @@ -706,6 +706,55 @@ def test_grow_by_factor_and_wait_visible(self): self.assertEqual(FakeArrayPlugin.luns[lun["id"]]["size"], 15 * GIB) self.assertEqual(b.agent.rescan_block_device.call_count, 2) + def test_grow_to_a_size_is_idempotent(self): + a, b = _node("a"), _node("b") + db = self.use_db(FakeDB(_cluster(lun_size=10 * GIB), [a, b])) + p = self.plugin(db.cluster) + la, lb = p.create_lun("a", 10 * GIB, {}), p.create_lun("b", 12 * GIB, {}) + da, db_ = _lun_device(a, la["id"], la["wwn"]), _lun_device(b, lb["id"], lb["wwn"]) + a.agent.rescan_block_device.return_value = ({"size": 15 * GIB}, None) + b.agent.rescan_block_device.return_value = ({"size": 15 * GIB}, None) + out = array_controller.grow_luns(CID, size=15 * GIB) + self.assertEqual(out["errors"], {}) + self.assertEqual((FakeArrayPlugin.luns[la["id"]]["size"], FakeArrayPlugin.luns[lb["id"]]["size"]), + (15 * GIB, 15 * GIB)) + self.assertEqual(out["luns"][db_.get_id()]["old_size"], 12 * GIB) + # Nodes added later get LUNs of the new size. + self.assertEqual(db.cluster.array_lun_size, 15 * GIB) + # Repeated (after a failure elsewhere): nothing is resized again. + FakeArrayPlugin.calls.clear() + out = array_controller.grow_luns(CID, size=15 * GIB) + self.assertEqual(out["errors"], {}) + self.assertFalse([c for c in FakeArrayPlugin.calls if c[0] == "resize"]) + self.assertEqual(FakeArrayPlugin.luns[la["id"]]["size"], 15 * GIB) + self.assertEqual(da.get_id() in out["luns"], True) + + def test_a_failed_growth_to_a_size_keeps_the_lun_size(self): + a = _node("a") + db = self.use_db(FakeDB(_cluster(lun_size=10 * GIB), [a])) + p = self.plugin(db.cluster) + lun = p.create_lun("a", 10 * GIB, {}) + _lun_device(a, lun["id"], lun["wwn"]) + a.agent.rescan_block_device.return_value = ({"size": 10 * GIB}, None) + out = array_controller.grow_luns(CID, size=15 * GIB, visible_timeout=0) + self.assertTrue(out["errors"]) + self.assertEqual(db.cluster.array_lun_size, 10 * GIB) + + def test_factor_or_size(self): + self.use_db(FakeDB(_cluster(), [_node("a")])) + for kw in ({}, {"factor": 2, "size": GIB}, {"size": 0}, {"size": "x"}): + with self.subTest(kw=kw), self.assertRaises(ValueError): + array_controller.grow_luns(CID, **kw) + + def test_grow_devices_passes_the_size(self): + self._prechecked() + with patch.object(array_controller, "grow_luns", return_value={"luns": {}, "errors": {}}) as grow_luns, patch.object(device_controller, "_grow_cluster_devices", + return_value={"grown": False, "factor": 1.0, "devices": [], "errors": {}, + "reason": "", "shrunk": []}): + out = device_controller.grow_cluster_devices(CID, lun_size=6 << 40) + grow_luns.assert_called_once_with(CID, None, size=6 << 40) + self.assertEqual(out["errors"], {}) + def test_refusals_and_errors(self): a = _node("a") db = self.use_db(FakeDB(_cluster(), [a])) From 811da71625163cafa3b023196119cdcbb4d75f5e Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 21:04:30 +0100 Subject: [PATCH 43/50] feat(array): the local key-store opt-in travels with the array registration register_array / POST /clusters/{id}/arrays take allow_local_kms_credentials (true/false; omitted = unchanged): a cluster without an external KMS can be told to keep the wrapped array login in its local key store with the registration itself, so an operator-managed cluster never needs the CLI for it. Applied to the store of that registration and saved only with it. Co-Authored-By: Claude Opus 5.5 --- .../controllers/array_controller.py | 14 +++++++++++++- simplyblock_web/api/v2/cluster/array.py | 7 ++++++- tests/unit/test_array_journal.py | 18 ++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/controllers/array_controller.py b/simplyblock_core/controllers/array_controller.py index c384a2caa8..5b294db486 100644 --- a/simplyblock_core/controllers/array_controller.py +++ b/simplyblock_core/controllers/array_controller.py @@ -163,7 +163,7 @@ def array_view(cluster, capabilities=None) -> dict: def register_array(cluster_id, plugin, endpoint="", options=None, credentials=None, lun_size=0, credential_ref=None, journal=JOURNAL_LOCAL, journal_lun_size=0, - journal_options=None) -> dict: + journal_options=None, allow_local_kms_credentials=None) -> dict: """Register (or re-register, e.g. to rotate credentials) the storage array of a multi-attach cluster. @@ -188,6 +188,12 @@ def register_array(cluster_id, plugin, endpoint="", options=None, credentials=No bytes (0 = 3% of ``lun_size``, at least 2 GiB). A change applies to nodes added afterwards. + ``allow_local_kms_credentials`` (True / False; None leaves it as it is) + sets ``Cluster.allow_local_kms_credentials``, the opt-in to keeping the + wrapped credentials in the local key store of a cluster without an + external KMS (test environments). It is saved only with a registration + that succeeds. + Returns the array view (``array_view``) with the plugin's capabilities. Raises: @@ -268,6 +274,10 @@ def register_array(cluster_id, plugin, endpoint="", options=None, credentials=No if max_att and max_att < needed: raise ValueError(f"the array presents a LUN to at most {max_att} hosts; the cluster needs {needed}") + if allow_local_kms_credentials is not None: + # Asked for with the registration, so it applies to this very store; + # the record keeps it only if the registration is saved below. + cluster.allow_local_kms_credentials = bool(allow_local_kms_credentials) if creds and not ref: try: ref = creds_mod.CredentialProvider(cluster).store(creds, context).model_dump() @@ -288,6 +298,8 @@ def mutate(fresh): fresh.array_journal = journal fresh.array_journal_lun_size = int(journal_lun_size or 0) fresh.array_journal_options = journal_opts + if allow_local_kms_credentials is not None: + fresh.allow_local_kms_credentials = bool(allow_local_kms_credentials) db.atomic_update(cluster, mutate) if legacy_secret: diff --git a/simplyblock_web/api/v2/cluster/array.py b/simplyblock_web/api/v2/cluster/array.py index 69337a24ac..be53e99421 100644 --- a/simplyblock_web/api/v2/cluster/array.py +++ b/simplyblock_web/api/v2/cluster/array.py @@ -32,6 +32,10 @@ class ArrayParams(BaseModel): '{"kind": "vault-kv", "mount": ..., "path": ..., "version": ...} or ' '{"kind": "kmip-secret", "uid": ..., "field": ...}'))] = Field(default_factory=dict) lun_size: Annotated[util.Size, Field(description="Size of the LUN created for each added node; 0 = none")] = 0 + allow_local_kms_credentials: Annotated[bool | None, Field(description=( + "Opt in (true) or out (false) of keeping the credentials in the cluster's local key store, " + "effectively unencrypted, when it has no external KMS (test environments); omitted, unchanged" + ))] = None journal: Annotated[Literal["local", "lun"], Field(description=( "Where each node's journal is kept: local (a disk or partition of its host) or lun " "(a LUN of its own, created next to its data LUN; needs lun_size)"))] = "local" @@ -66,7 +70,8 @@ def register(cluster: Cluster, parameters: ArrayParams) -> ArrayDTO: cluster.get_id(), parameters.plugin, parameters.endpoint, parameters.options, parameters.credentials, parameters.lun_size, credential_ref=parameters.credential_ref or None, journal=parameters.journal, - journal_lun_size=parameters.journal_lun_size, journal_options=parameters.journal_options or None) + journal_lun_size=parameters.journal_lun_size, journal_options=parameters.journal_options or None, + allow_local_kms_credentials=parameters.allow_local_kms_credentials) except ValueError as e: raise HTTPException(400, str(e)) return ArrayDTO(**view) diff --git a/tests/unit/test_array_journal.py b/tests/unit/test_array_journal.py index 74c1c80cbe..8a15b11395 100644 --- a/tests/unit/test_array_journal.py +++ b/tests/unit/test_array_journal.py @@ -96,6 +96,24 @@ def test_delete_resets_and_counts_journal_luns(self): self.assertEqual((db.cluster.array_journal, db.cluster.array_journal_options), ("local", {})) +class TestLocalKeyStoreOptIn(_Case): + """The opt-in to the local key store travels with the registration, so an + operator-managed cluster never needs the CLI for it.""" + + def test_opt_in_with_the_registration(self): + db = self.use_db(FakeDB(_cluster(plugin=""), [])) + db.cluster.allow_local_kms_credentials = False + with self.assertRaisesRegex(ValueError, "could not be stored"): + array_controller.register_array(CID, PLUGIN, credentials={"p": "x"}) + self.assertFalse(db.cluster.allow_local_kms_credentials) + array_controller.register_array(CID, PLUGIN, credentials={"p": "x"}, allow_local_kms_credentials=True) + self.assertTrue(db.cluster.allow_local_kms_credentials) + self.assertEqual(db.cluster.array_credential_ref["kind"], "managed") + # Omitted: unchanged. + array_controller.register_array(CID, PLUGIN, credentials={"p": "y"}) + self.assertTrue(db.cluster.allow_local_kms_credentials) + + class TestSize(unittest.TestCase): def test_default_is_3_percent_at_least_2_gib_whole_gib(self): From fb70786ca1c83b12306ab8cedc60c453f06e0329 Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 21:15:41 +0100 Subject: [PATCH 44/50] multi-attach takeover: a dead standby costs ~1 s; warm standbys sized at add 2026-11-20 FTT2 bed, cases 7 (reboot sn2 + hard reset sn3) and 8 (hard reset sn3 + network cut sn4): the takeover chose the other lost node as standby. The standby failover worked, but the activation wait ran its full 5 s while every poll failed to connect, then an array fence (1 s) and a second reachability check (0.8 s) ran before the next standby: 13.1 s and 12.4 s stalls against the 11 s limit. - _wait_active: two consecutive connection errors raise StandbyUnreachable. - A standby that is gone is left at once: no retry on it and no array fence first; the next standby opens with its own key and preempts by reservation (the array fence still follows in the background). A standby that answers but did not take the reservation is fenced at the array and retried as before. - The export's cleanup reports an unreachable standby as StandbyUnreachable. Warm standby: add-node sized the SPDK memory for the node's own LUN only, so every warm standby was refused for budget until the node's first restart (warm-standby bed). Add-node now sizes for FTT standby LUNs of the node's own size (LUN sizes are uniform); multi_attach.warm_standby_cluster(). Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/controllers/lun_takeover.py | 38 ++++++++++++++++---- simplyblock_core/multi_attach.py | 5 +++ simplyblock_core/storage_node_ops.py | 24 ++++++++----- tests/unit/test_lun_takeover.py | 26 ++++++++++++++ 4 files changed, 79 insertions(+), 14 deletions(-) diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index 7afabe48bf..b505ff3266 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -97,6 +97,17 @@ class TakeoverError(Exception): """A takeover step failed; the message says which and why.""" +class StandbyUnreachable(TakeoverError): + """The standby's SPDK stopped answering during the export: the next + standby is tried at once (no retry on it, no array fence first).""" + + +#: Consecutive connection errors while waiting for an activation after which +#: the standby counts as gone (the poll interval is 50 ms, each call times out +#: after 1 s). +ACTIVATION_CONNECTION_ERRORS = 2 + + def _local_lock(device_id) -> threading.Lock: with _local_locks_guard: return _local_locks.setdefault(device_id, threading.Lock()) @@ -275,14 +286,18 @@ def _take_over_locked(db, cluster, device_id, failed_node_id, reason, deadline) break except TakeoverError as e: sid = standby.get_id() - if by_reservation: - # The reservation was not taken: fence at the array. + gone = isinstance(e, StandbyUnreachable) + if by_reservation and not gone: + # The standby answers but did not take the reservation: + # fence at the array. A standby that is gone fenced + # nothing; the next one opens with its own key and + # preempts, so it is tried at once. logger.warning("LUN takeover of %s: fence by reservation failed (%s); " "fencing at the array", device_id, e) by_reservation = False _fence(cluster, dev, failed) _record_fenced(db, owner, device_id, failed_node_id) - if sid not in retried and _reachable(standby): + if not gone and sid not in retried and _reachable(standby): retried.add(sid) continue tried.add(sid) @@ -441,11 +456,18 @@ def _wait_active(rpc, alceml, deadline): holds the reservation and serves. Raises TakeoverError.""" limit = min(deadline, time.monotonic() + constants.MULTI_ATTACH_ACTIVATION_WAIT_SEC) last = None + unreachable = 0 while True: try: last = rpc.alceml_get_role(alceml) or {} + unreachable = 0 except Exception as e: # noqa: BLE001 last = {"error": str(e)} + unreachable += 1 + if unreachable >= ACTIVATION_CONNECTION_ERRORS: + # The standby is gone (2026-11-20 FTT2 bed: a rebooting + # standby cost the whole wait, 5 s, of a 13 s stall). + raise StandbyUnreachable(f"alceml {alceml}: SPDK not answering ({e})") if last.get("role") == "active" and last.get("state") == "active": logger.info("LUN takeover: %s holds the reservation (epoch %s, activation %s ms)", alceml, last.get("epoch"), last.get("last_activation_ms")) @@ -676,14 +698,18 @@ def _export_on_standby(db, cluster, standby, dev, path, deadline, allow_warm=Tru if not rpc.nvmf_subsystem_add_ns(nqn, pt, multi_attach.namespace_uuid(dev), nguid=multi_attach.namespace_nguid(dev)): raise TakeoverError(f"namespace {pt} not added to {nqn} on {sid}") - except Exception: + except StandbyUnreachable: + logger.warning("LUN takeover cleanup on %s skipped: SPDK not answering (%s)", + sid, [name for _, name in created]) + raise + except Exception as e: if created and not _reachable(standby): # Its SPDK is gone: what it held went with it (each call would # only wait out the retries). logger.warning("LUN takeover cleanup on %s skipped: SPDK not answering (%s)", sid, [name for _, name in created]) - else: - _teardown(rpc, sid, created) + raise StandbyUnreachable(f"standby {sid}: {e}") from e + _teardown(rpc, sid, created) raise aliases = (pt_info or {}).get("aliases") or [] return { diff --git a/simplyblock_core/multi_attach.py b/simplyblock_core/multi_attach.py index f7fcbc378e..aa7e35ed37 100644 --- a/simplyblock_core/multi_attach.py +++ b/simplyblock_core/multi_attach.py @@ -366,6 +366,11 @@ def warm_standby_devices(cluster, node_id, nodes, statuses=_WARM_STANDBY_DEVICE_ return out +def warm_standby_cluster(cluster) -> bool: + """A multi-attach cluster created with warm standbys.""" + return is_multi_attach(cluster) and getattr(cluster, "multi_attach_warm_standby", False) is True + + def warm_standby_capacity(cluster, node_id, nodes) -> int: """Raw capacity (bytes) of the LUNs ``node_id`` keeps (or will keep, once they are back online) a warm standby of: the capacity whose alceml page diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index dd37319b75..8c6b818c08 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4411,6 +4411,18 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, lun_capacity += array_controller.journal_lun_size(cluster) minimum_sys_memory = max(minimum_sys_memory, utils.node_config_min_sys_memory(node_config, lun_capacity)) + if multi_attach.warm_standby_cluster(cluster): + # The node will also hold the page map of the LUNs it is a + # warm standby of: FTT of them, each the size of its own + # (LUN sizes are uniform; growth is uniform-only). Sized + # here, the standbys open right away instead of waiting + # for the node's first restart (2026-11-20 bed: every + # standby refused for budget after add-node). + own = sum(int(e.get("size") or 0) + for e in node_config.get("lblk_devices") or []) + lun_capacity + ftt = max(1, int(getattr(cluster, "max_fault_tolerance", 1) or 1)) + minimum_sys_memory = max(minimum_sys_memory, utils.node_config_min_sys_memory( + node_config, lun_capacity + ftt * own)) else: ssd_pcis = node_config.get("ssd_pcis") or [] capacity = utils.nvme_capacity_from_node_info(node_info.get("nvme_devices"), ssd_pcis) @@ -4421,14 +4433,10 @@ def add_node(cluster_id, node_addr, iface_name, data_nics_list, "devices visible (unbind them from SPDK) or pass spdk_sys_mem.", ssd_pcis, node_addr) return False minimum_sys_memory = max(minimum_sys_memory, utils.sys_memory_for_capacity(capacity)) - # Warm-standby multi-attach clusters (Cluster.multi_attach_warm_standby): - # the node will also hold the page map of every LUN it becomes a warm - # standby of, i.e. sys_memory_for_capacity(own + standby LUN capacity). - # That set (the LUNs of the nodes whose LVS secondary/tertiary this node - # becomes) is not known yet here; it is counted from the node's first - # restart on (_warm_standby_capacity), and until then - # lun_takeover.ensure_warm_standbys opens no standby beyond this budget - # and raises a cluster event instead. + # Warm-standby multi-attach clusters: the lblk branch above sizes for + # FTT standby LUNs of the node's own size; from the first restart on + # the actual set counts (_warm_standby_capacity), and + # lun_takeover.ensure_warm_standbys never opens beyond the budget. max_lvol = node_config.get("max_lvol") # Clamp rather than fail: a config file generated before the cap was # enforced (or hand-edited) can carry a larger value, and refusing the diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index dcaf6b198c..fe5c83b427 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -1108,6 +1108,32 @@ def b_dies(name): # No cleanup calls into the dead standby. self.assertEqual(b.spdk.ops("bdev_alceml_delete"), []) + def test_dead_standby_costs_no_activation_wait(self): + # 2026-11-20 FTT2 bed, case 7 again: the whole 5 s activation wait + # and an array fence were spent on a rebooting standby (13 s stall). + db, _ = self._bed() + b, c = db.nodes["node-b"], db.nodes["node-c"] + + def b_gone(name): + b.spdk.down = True + raise ConnectionError("Could not reach remote") + b.spdk.alceml_get_role = b_gone + self._roles(c, {"role": "active", "state": "active", "epoch": 3}) + fenced_before_c = [] + create_c = c.spdk.bdev_alceml_create + c.spdk.bdev_alceml_create = lambda *a, **k: ( + fenced_before_c.append(list(FakeArrayPlugin.calls)), create_c(*a, **k))[1] + t0 = time.monotonic() + with patch.object(constants, "MULTI_ATTACH_ACTIVATION_WAIT_SEC", 5): + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertLess(time.monotonic() - t0, 2.0) + self.assertEqual(self.dev(db).serving_node(), "node-c") + # node-c preempted by reservation: no array fence before it opened. + self.assertEqual(fenced_before_c, [[]]) + self.assertEqual(len(b.spdk.ops("bdev_alceml_create")), 1) + self._wait_background_fence() + self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-a")]) + def test_no_standby_left_fails(self): db, _ = self._bed() def dies(spdk): From 44d4dfc562b643223d67a84b61506207d358f1bd Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 22:02:40 +0100 Subject: [PATCH 45/50] multi-attach: refuse warm standby until its stale-data bug is fixed 2026-11-20 warm-standby FTT2 bed (fb70786c): a network cut of the serving node made the takeover reuse the warm standby on sn1 (activation 141 ms); fio crc32c verification then failed (EILSEQ) on 4 of 5 volumes. The same case with a cold takeover on the same build verified clean. The SPDK log of the activated instance was lost (restart), so the cause in the alceml warm refresh/activation is not confirmed yet. cluster create/add refuse --multi-attach-warm-standby unless the control plane runs with SIMPLYBLOCK_ALLOW_WARM_STANDBY=1 (repro beds only). The cold takeover, the default, is unchanged. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/cluster_ops.py | 5 +++++ simplyblock_core/constants.py | 6 ++++++ tests/unit/test_lun_warm_standby.py | 14 +++++++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 29c5a2dded..26e0e93388 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -472,6 +472,11 @@ def _validate_warm_standby(multi_attach, warm_standby): option only.""" if warm_standby and not multi_attach: raise ValueError("multi_attach_warm_standby requires multi_attach") + if warm_standby and not constants.MULTI_ATTACH_WARM_STANDBY_ALLOWED: + raise ValueError( + "multi_attach_warm_standby is disabled: on the 2026-11-20 bed a takeover onto a warm " + "standby returned stale data (fio crc32c verify failures); it stays off until the " + "alceml warm refresh is fixed") def _validate_integrity_flags(inline_checksum, atomic_4k): diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index e804e84ce6..a71393ff9d 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -1370,6 +1370,12 @@ def get_config_var(name, default=None): # How often the storage-node monitor reconciles a node's warm standbys # (lun_takeover.ensure_warm_standbys), per node. MULTI_ATTACH_WARM_STANDBY_INTERVAL_SEC = 30 +# Warm standby is refused at cluster create: on the 2026-11-20 FTT2 bed a +# takeover onto a warm standby (activation 141 ms) was followed by crc32c +# verify failures on 4 of 5 volumes; the cold takeover on the same build and +# case verified clean. Test beds that reproduce it set +# SIMPLYBLOCK_ALLOW_WARM_STANDBY=1 on the control plane. +MULTI_ATTACH_WARM_STANDBY_ALLOWED = os.environ.get("SIMPLYBLOCK_ALLOW_WARM_STANDBY") == "1" # Client keep-alive on multi-attach clusters: a node lost without closing its # connections (hard reset, network cut) leaves its own volumes' paths dead # until the host's keep-alive notices, 1-2 intervals: 8 s took 10-14 s on the diff --git a/tests/unit/test_lun_warm_standby.py b/tests/unit/test_lun_warm_standby.py index bec815a796..545b9f2941 100644 --- a/tests/unit/test_lun_warm_standby.py +++ b/tests/unit/test_lun_warm_standby.py @@ -417,11 +417,19 @@ def test_helper(self): def test_cluster_flag_needs_multi_attach(self): self.assertFalse(Cluster().multi_attach_warm_standby) - with self.assertRaises(ValueError): - cluster_ops._validate_warm_standby(False, True) - cluster_ops._validate_warm_standby(True, True) + with patch.object(constants, "MULTI_ATTACH_WARM_STANDBY_ALLOWED", True): + with self.assertRaises(ValueError): + cluster_ops._validate_warm_standby(False, True) + cluster_ops._validate_warm_standby(True, True) cluster_ops._validate_warm_standby(False, False) + def test_refused_while_disabled(self): + # 2026-11-20 bed: stale data after a warm takeover. + with patch.object(constants, "MULTI_ATTACH_WARM_STANDBY_ALLOWED", False): + with self.assertRaises(ValueError) as cm: + cluster_ops._validate_warm_standby(True, True) + self.assertIn("disabled", str(cm.exception)) + def test_cli_flag(self): import argparse From 7406ebdf547b3e62479198878eaa6ed171988cfa Mon Sep 17 00:00:00 2001 From: michael Date: Fri, 20 Nov 2026 22:53:00 +0100 Subject: [PATCH 46/50] multi-attach: full node writes keep the LUN ownership fields 2026-11-20 FTT2 bed (fb70786c), net cut sn0 + SPDK kill sn1: the reintegration unfenced the cut node at 21:32:37.754 and re-attached its LUN; the owner (sn1, restarting) then wrote its full record five times in 1.4 s from a copy loaded before the unfence, restoring fenced_node_ids. Record "fenced" + host showing the LUN = "operator unfences": the node stayed fenced and the case never healed. serving_node_id, fenced_node_ids and array_attached_node_ids of a multi-attach device are changed only by atomic_update (takeover, fence, unfence, attachment reconcile). A full-object StorageNode write now keeps their stored values (BaseModel._write_tx -> StorageNode.merge_on_full_write) and logs what it kept. Only nodes with multi-attach devices pay the stored-record read; every other node write is unchanged. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/models/base_model.py | 7 ++ simplyblock_core/models/storage_node.py | 40 +++++++ tests/unit/test_node_full_write_lun_fields.py | 108 ++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 tests/unit/test_node_full_write_lun_fields.py diff --git a/simplyblock_core/models/base_model.py b/simplyblock_core/models/base_model.py index 13f3243a7f..8d461350fb 100644 --- a/simplyblock_core/models/base_model.py +++ b/simplyblock_core/models/base_model.py @@ -503,6 +503,13 @@ def _apply_index_diff(tr, model_cls, index_list, old_keys, obj, *, @staticmethod def _write_tx(tr, key, value, model_cls, obj, index_list, rollup_key, version_key): + merge = getattr(model_cls, "merge_on_full_write", None) + if merge is not None and obj.needs_full_write_merge(): + # Fields only atomic_update may change keep their stored value: a + # full-object write of a stale copy must not revert them. + raw = tr.get(key).wait() + if raw is not None and raw.present(): + value = merge(json.loads(bytes(raw)), value) if index_list: BaseModel._apply_index_diff( tr, model_cls, index_list, diff --git a/simplyblock_core/models/storage_node.py b/simplyblock_core/models/storage_node.py index 898802adc5..f4d2b3bacf 100644 --- a/simplyblock_core/models/storage_node.py +++ b/simplyblock_core/models/storage_node.py @@ -1,4 +1,5 @@ import hashlib +import json import time from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, ClassVar @@ -42,6 +43,45 @@ class StorageNode(BaseNodeObject): _WATCHED = True + #: Multi-attach LUN ownership fields of a device. Only atomic_update + #: changes them (lun_takeover: takeover, fence, unfence; array_controller: + #: attachment reconcile); a full-object write keeps their stored values. + #: A restart's full writes from a copy loaded before an unfence restored + #: fenced_node_ids, and the reintegrated node stayed fenced for good + #: (2026-11-20 FTT2 bed, net cut + SPDK kill of the owner). + LUN_OWNERSHIP_FIELDS = ("serving_node_id", "fenced_node_ids", "array_attached_node_ids") + + def needs_full_write_merge(self) -> bool: + """Only nodes with multi-attach devices pay the stored-record read + (every other node's write is unchanged).""" + return any(getattr(d, "multi_attach", False) is True for d in self.nvme_devices or []) + + @staticmethod + def merge_on_full_write(stored: dict, value: bytes) -> bytes: + """``value`` (a full-object write) with the stored LUN ownership + fields of every multi-attach device present in both.""" + old = {d.get("uuid"): d for d in stored.get("nvme_devices") or [] + if isinstance(d, dict) and d.get("multi_attach") is True} + if not old: + return value + new = json.loads(value) + kept = [] + for d in new.get("nvme_devices") or []: + o = old.get(d.get("uuid")) if isinstance(d, dict) else None + if o is None: + continue + for f in StorageNode.LUN_OWNERSHIP_FIELDS: + if f in o and d.get(f) != o.get(f): + kept.append(f"{d.get('uuid')}.{f}: {d.get(f)!r} -> kept {o.get(f)!r}") + d[f] = o[f] + if not kept: + return value + from simplyblock_core import utils + utils.get_logger(__name__).warning( + "[NODE-WRITE] full-object write of %s from a stale copy: LUN ownership kept from the " + "stored record (%s)", new.get("uuid"), "; ".join(kept)) + return json.dumps(new).encode() + # A JM built on a whole device inherits that device's uuid # (storage_node_ops._create_jm_stack_on_device), so a bare device id does # not say which kind of device it names. The kind is the second segment of diff --git a/tests/unit/test_node_full_write_lun_fields.py b/tests/unit/test_node_full_write_lun_fields.py new file mode 100644 index 0000000000..c329c52613 --- /dev/null +++ b/tests/unit/test_node_full_write_lun_fields.py @@ -0,0 +1,108 @@ +"""A full-object write of a StorageNode keeps the stored multi-attach LUN +ownership fields of its devices (serving_node_id, fenced_node_ids, +array_attached_node_ids): only atomic_update changes them. + +2026-11-20 FTT2 bed (net cut sn0 + SPDK kill sn1): the reintegration +unfenced a node at 21:32:37.754; the owner's restart, holding a copy loaded +before that, wrote its full record five times in the next 1.4 s and restored +fenced_node_ids. The node stayed fenced and the case never healed.""" + +import json +import unittest + +from simplyblock_core.models.base_model import BaseModel +from simplyblock_core.models.nvme_device import NVMeDevice +from simplyblock_core.models.storage_node import StorageNode + + +def _node(fenced, serving="", attached=(), status=StorageNode.STATUS_ONLINE, multi=True): + n = StorageNode() + n.uuid = "owner" + n.status = status + d = NVMeDevice() + d.uuid = "dev-1" + d.node_id = "owner" + d.multi_attach = multi + d.fenced_node_ids = list(fenced) + d.serving_node_id = serving + d.array_attached_node_ids = list(attached) + n.nvme_devices = [d] + return n + + +class _Tr: + def __init__(self, stored): + self.data = {b"k": json.dumps(stored.to_dict(unwrap_secrets=True)).encode()} if stored else {} + + def get(self, key): + raw = self.data.get(key) + + class _V: + def wait(self_inner): + return self_inner + + def present(self_inner): + return raw is not None + + def __bytes__(self_inner): + return raw + return _V() + + def set(self, key, value): + self.data[key] = value + + +def _write(stored, stale): + tr = _Tr(stored) + BaseModel._write_tx(tr, b"k", json.dumps(stale.to_dict(unwrap_secrets=True)).encode(), + StorageNode, stale, [], None, None) + return StorageNode().from_dict(json.loads(tr.data[b"k"])).nvme_devices[0] + + +class TestFullWriteKeepsLunOwnership(unittest.TestCase): + + def test_stale_copy_does_not_refence(self): + stored = _node(fenced=[], serving="node-b", attached=["node-a", "node-b"]) + stale = _node(fenced=["node-a"], serving="node-b", attached=["node-b"], + status=StorageNode.STATUS_RESTARTING) + dev = _write(stored, stale) + self.assertEqual(dev.fenced_node_ids, []) + self.assertEqual(dev.array_attached_node_ids, ["node-a", "node-b"]) + + def test_stale_copy_does_not_undo_a_takeover(self): + dev = _write(_node(fenced=["owner"], serving="node-b"), _node(fenced=[], serving="")) + self.assertEqual(dev.serving_node_id, "node-b") + self.assertEqual(dev.fenced_node_ids, ["owner"]) + + def test_other_fields_are_written(self): + stored = _node(fenced=[]) + stale = _node(fenced=[], status=StorageNode.STATUS_RESTARTING) + stale.nvme_devices[0].nvmf_nqn = "nqn.new" + tr = _Tr(stored) + BaseModel._write_tx(tr, b"k", json.dumps(stale.to_dict(unwrap_secrets=True)).encode(), + StorageNode, stale, [], None, None) + n = StorageNode().from_dict(json.loads(tr.data[b"k"])) + self.assertEqual(n.status, StorageNode.STATUS_RESTARTING) + self.assertEqual(n.nvme_devices[0].nvmf_nqn, "nqn.new") + + def test_new_record_is_written_as_is(self): + dev = _write(None, _node(fenced=["x"], serving="y")) + self.assertEqual((dev.fenced_node_ids, dev.serving_node_id), (["x"], "y")) + + def test_regular_node_is_untouched(self): + # No multi-attach device: no stored-record read, the write as before. + stored, stale = _node(fenced=[], multi=False), _node(fenced=["x"], multi=False) + self.assertFalse(stale.needs_full_write_merge()) + tr = _Tr(stored) + reads = [] + get = tr.get + tr.get = lambda key: (reads.append(key), get(key))[1] + BaseModel._write_tx(tr, b"k", json.dumps(stale.to_dict(unwrap_secrets=True)).encode(), + StorageNode, stale, [], None, None) + self.assertEqual(reads, []) + dev = StorageNode().from_dict(json.loads(tr.data[b"k"])).nvme_devices[0] + self.assertEqual(dev.fenced_node_ids, ["x"]) + + +if __name__ == "__main__": + unittest.main() From f478e203275ec2ff2ebf3f2d990b0202a48f9dc5 Mon Sep 17 00:00:00 2001 From: michael Date: Sat, 21 Nov 2026 00:12:45 +0100 Subject: [PATCH 47/50] fix(multi-attach): lun_watchdog is not a service; clear two CodeQL alerts - services/__main__.py: lun_watchdog runs as a thread inside storage_node_monitor and has no main(), so it is listed as a library (test_every_service_defines_main failed on CI for 'lun-watchdog'). - credentials.py: the store log line gives the number of fields, not their names (CodeQL clear-text logging of sensitive data). - storage_arrays/testing: the mock array's TLS server refuses TLS < 1.2. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/credentials.py | 4 ++-- simplyblock_core/services/__main__.py | 1 + simplyblock_core/storage_arrays/testing/__init__.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/simplyblock_core/credentials.py b/simplyblock_core/credentials.py index 066fcf2a1b..070220625c 100644 --- a/simplyblock_core/credentials.py +++ b/simplyblock_core/credentials.py @@ -215,8 +215,8 @@ def store( wrapped = kms.wrap_secret(kek_name, secret_fields, context) except KMSException as e: raise CredentialError(f"Storing credential failed: {e.message}") from e - logger.info("Stored managed credential (backend=%s, fields=%s, context=%s)", - backend, sorted(secret_fields), context) + logger.info("Stored managed credential (backend=%s, %d fields, context=%s)", + backend, len(secret_fields), context) return ManagedCredentialRef(backend=backend, kek_name=kek_name, wrapped=wrapped) # -- Resolve ------------------------------------------------------------ diff --git a/simplyblock_core/services/__main__.py b/simplyblock_core/services/__main__.py index e7637f5014..86c5e6c3ee 100644 --- a/simplyblock_core/services/__main__.py +++ b/simplyblock_core/services/__main__.py @@ -23,6 +23,7 @@ # advertise names that do nothing when run. _LIBRARIES = frozenset({ "hub_controller_manager", + "lun_watchdog", # a thread inside storage_node_monitor "migration_task_common", "replication_final_step", "task_runner_base", diff --git a/simplyblock_core/storage_arrays/testing/__init__.py b/simplyblock_core/storage_arrays/testing/__init__.py index cf82bb245e..35f5276847 100644 --- a/simplyblock_core/storage_arrays/testing/__init__.py +++ b/simplyblock_core/storage_arrays/testing/__init__.py @@ -314,6 +314,7 @@ def log_message(self, fmt, *args): # quieter default logging certfile, keyfile = f"{tmp}/cert.pem", f"{tmp}/key.pem" _self_signed(host, certfile, keyfile) ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 ctx.load_cert_chain(certfile, keyfile) server.socket = ctx.wrap_socket(server.socket, server_side=True) server.certfile = certfile # type: ignore[attr-defined] From f1d109fe531901418acd5fbc412d37b909fd7759 Mon Sep 17 00:00:00 2001 From: michael Date: Sat, 21 Nov 2026 00:58:56 +0100 Subject: [PATCH 48/50] multi-attach takeover: probe standby candidates at once, bounded 2026-11-21 FTT2 bed (7406ebdf), network cuts of sn4 and sn0: two of the three LUNs sn4 served moved within 1.5 s; the third chose its standby one candidate after the other, and the first, sn0, had just lost its network. Its agent query (15 s client timeout, default retries) held the choice until the 90 s takeover deadline; the takeover failed, the LUN went unavailable and the 1+0 cluster suspended. - _choose_standby probes every eligible candidate in parallel; each agent query is bounded by STANDBY_PROBE_TIMEOUT_SEC (3 s) without retries, and so is the wait for them. - Candidate order still wins; a candidate that reads the LUN waits at most STANDBY_ORDER_GRACE_SEC (0.5 s) for an earlier one that has not answered. Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/controllers/lun_takeover.py | 72 +++++++++++++++++--- tests/unit/test_lun_takeover.py | 22 ++++++ 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index b505ff3266..a95be1303e 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -106,6 +106,11 @@ class StandbyUnreachable(TakeoverError): #: the standby counts as gone (the poll interval is 50 ms, each call times out #: after 1 s). ACTIVATION_CONNECTION_ERRORS = 2 +#: Bound of one standby candidate's LUN inventory query (no retries). +STANDBY_PROBE_TIMEOUT_SEC = 3.0 +#: How long a candidate that reads the LUN waits for an earlier candidate +#: (candidate order) whose probe has not answered yet. +STANDBY_ORDER_GRACE_SEC = 0.5 def _local_lock(device_id) -> threading.Lock: @@ -490,7 +495,8 @@ def _mut(n): # --------------------------------------------------------- 2 + 3. standby def _choose_standby(owner, dev, failed_node_id, nodes_by_id, deadline, exclude=()): - reasons = [] + reasons: list = [] + eligible: list = [] for nid in multi_attach.standby_candidates(owner, failed_node_id): node = nodes_by_id.get(nid) if node is None or nid in exclude: @@ -504,14 +510,61 @@ def _choose_standby(owner, dev, failed_node_id, nodes_by_id, deadline, exclude=( if _transport(node) is None: reasons.append(f"{nid}: no active fabric") continue - path, why = _probe(node, dev, deadline) - if path: - logger.info("LUN %s: standby %s reads it at %s", dev.lun_wwn, nid, path) - return node, path, "" - reasons.append(f"{nid}: {why}") - if time.monotonic() > deadline: - reasons.append("deadline passed") + eligible.append((nid, node)) + if not eligible: + return None, None, "; ".join(reasons) or "no standby candidate" + # Every candidate is probed at once, each call bounded and without + # retries; the first in candidate order that reads the LUN wins. One + # after the other, a candidate that had just lost its network held the + # choice until the takeover deadline (2026-11-20 FTT2 bed, network cuts + # of two nodes: the LUN went unavailable and the cluster suspended). + pool = ThreadPoolExecutor(max_workers=len(eligible), thread_name_prefix=f"probe-{dev.get_id()[:8]}") + probes = [(nid, node, pool.submit(_probe, node, dev, deadline)) for nid, node in eligible] + pool.shutdown(wait=False) + # The wait is bounded too (a probe that ignores its client timeout must + # not hold the choice). Candidate order wins, but a candidate that already + # reads the LUN is not kept waiting more than STANDBY_ORDER_GRACE_SEC for + # an earlier one that does not answer. + now = time.monotonic() + probes_done_by = min(deadline, now + STANDBY_PROBE_TIMEOUT_SEC + 0.5) + first_ready_at = None + while True: + now = time.monotonic() + best = None + pending_before = False + for nid, node, fut in probes: + if not fut.done(): + pending_before = True + continue + try: + path, _why = fut.result() + except Exception: # noqa: BLE001 + path = None + if path: + best = (nid, node, path) + break + if best and not pending_before: break + if best and first_ready_at is None: + first_ready_at = now + if best and now - first_ready_at >= STANDBY_ORDER_GRACE_SEC: + break + if now >= probes_done_by or all(f.done() for _n, _o, f in probes): + break + time.sleep(0.02) + if best: + nid, node, path = best + logger.info("LUN %s: standby %s reads it at %s", dev.lun_wwn, nid, path) + return node, path, "" + for nid, _node, fut in probes: + if not fut.done(): + reasons.append(f"{nid}: probe did not answer within {STANDBY_PROBE_TIMEOUT_SEC:.0f}s") + continue + try: + _path, why = fut.result() + except Exception as e: # noqa: BLE001 + why = f"agent query failed: {e}" + reasons.append(f"{nid}: {why}") return None, None, "; ".join(reasons) or "no standby candidate" @@ -526,7 +579,8 @@ def _reachable(node) -> bool: def _probe(node, dev, deadline): try: - result, err = node.client(timeout=_remaining(deadline, 15)).lun_info(dev.lun_wwn) + result, err = node.client(timeout=min(_remaining(deadline, 15), STANDBY_PROBE_TIMEOUT_SEC), + retry=0, connect_retry=0).lun_info(dev.lun_wwn) except Exception as e: # noqa: BLE001 return None, f"agent query failed: {e}" if err: diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index fe5c83b427..257cb87156 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -1134,6 +1134,28 @@ def b_gone(name): self._wait_background_fence() self.assertEqual(FakeArrayPlugin.calls, [("fence", WWN, "host-node-a")]) + def test_a_hanging_candidate_does_not_hold_the_choice(self): + # 2026-11-20 FTT2 bed, network cuts of two nodes: the first candidate's + # agent did not answer, the choice waited until the 90 s deadline. + db, _ = self._bed() + b, c = db.nodes["node-b"], db.nodes["node-c"] + gate = threading.Event() + b.agent.lun_info.side_effect = lambda *_a, **_k: (gate.wait(10), (None, "gone"))[1] + self._roles(c, {"role": "active", "state": "active", "epoch": 3}) + t0 = time.monotonic() + with patch.object(lun_takeover, "STANDBY_PROBE_TIMEOUT_SEC", 3.0): + out = self.run_takeover(db, deadline_sec=4.0) + gate.set() + self.assertEqual(out, lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-c") + self.assertLess(time.monotonic() - t0, 2.0) + + def test_candidate_order_wins_when_several_answer(self): + db, _ = self._bed() + db.nodes["node-b"].spdk.alceml_get_role = lambda name: {"role": "active", "state": "active"} + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + self.assertEqual(self.dev(db).serving_node(), "node-b") + def test_no_standby_left_fails(self): db, _ = self._bed() def dies(spdk): From 76b9e7e718c76e8d51e2fd8fd81aeeedf0cc5514 Mon Sep 17 00:00:00 2001 From: michael Date: Sat, 21 Nov 2026 02:49:53 +0100 Subject: [PATCH 49/50] multi-attach: a dead NVMe controller does not hold a LUN fence 2026-11-21 FTT2 bed (f1d109fe), reboot sn2 + hard reset sn3: the takeover took 1.3 s and the background EBS detach of the reset node succeeded, but it landed while the host was booting. The NVMe controller reset failed and the kernel kept a dead 0-byte namespace (lsblk STATE dead). The agent's LUN inventory reported it, so the reintegration read "host still shows the LUN, an operator unfences" and the node stayed fenced: the case never healed. find_lun_devices skips a disk whose controller state is "dead": it can neither reach the array nor queue I/O for it. Live devices are reported as before. Co-Authored-By: Claude Opus 5.5 --- simplyblock_web/node_utils.py | 12 ++++++++++++ tests/unit/test_lun_takeover.py | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/simplyblock_web/node_utils.py b/simplyblock_web/node_utils.py index 9d9edd1aac..6c2c06342d 100644 --- a/simplyblock_web/node_utils.py +++ b/simplyblock_web/node_utils.py @@ -591,6 +591,12 @@ def scan_for_new_luns() -> dict: return {"scanned": scanned} +def _controller_dead(disk_name: str) -> bool: + """An NVMe namespace whose controller the kernel gave up on (state + ``dead``): no I/O can reach it or be queued on it.""" + return bool(disk_name) and _read_sysfs(f"/sys/block/{disk_name}/device/state") == "dead" + + def find_lun_devices(wwn: str) -> list[dict]: """The whole disks of this host that carry LUN ``wwn`` (multi-attach array LUN mode): every lsblk disk whose WWN, or serial (the inventory falls back @@ -610,6 +616,12 @@ def find_lun_devices(wwn: str) -> list[dict]: matched = [d for d in devices if d.get("type") in ("disk", "mpath") and want in (_normalize_wwn(d.get("wwn", "")), _normalize_wwn(d.get("serial", "")))] + # A disk whose NVMe controller is dead holds no path to the LUN and no + # I/O for it: the kernel keeps such a 0-byte remnant when the volume was + # detached while the host booted (2026-11-21 FTT2 bed: the fenced node, + # back from a hard reset, kept reporting the detached LUN and stayed + # fenced for good). + matched = [d for d in matched if not _controller_dead(d.get("name", ""))] # Several paths of one LUN under a multipath map: the map, not the paths. if any(d.get("type") == "mpath" for d in matched): matched = [d for d in matched if d.get("type") == "mpath"] diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index 257cb87156..ded9f65a1f 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -559,6 +559,18 @@ def test_find_lun_devices(self): self.assertTrue(out[0]["has_alceml_signature"]) sig.assert_called_once_with("sdb") + def test_find_lun_devices_skips_a_dead_controller(self): + # A volume detached while the host booted: the kernel keeps a 0-byte + # namespace of a dead controller. + from simplyblock_web import node_utils + inv = [{"type": "disk", "name": "nvme3n1", "device_path": "/dev/nvme3n1", "wwn": "", + "serial": "vol08dcbe488933595ca", "size": 0, "by_id_path": "/x"}] + states = {"/sys/block/nvme3n1/device/state": "dead"} + with patch.object(node_utils, "get_block_devices_info", return_value=inv), patch.object(node_utils, "_read_sysfs", side_effect=lambda p: states.get(p, "")), patch.object(node_utils, "simplyblock_data_signature", return_value=None): + self.assertEqual(node_utils.find_lun_devices("vol08dcbe488933595ca"), []) + states["/sys/block/nvme3n1/device/state"] = "live" + self.assertEqual(len(node_utils.find_lun_devices("vol08dcbe488933595ca")), 1) + # ------------------------------------------------------------------ monitor From fa6d6992220be21e76441d894ff31c9a1f2b4d80 Mon Sep 17 00:00:00 2001 From: michael Date: Sat, 21 Nov 2026 04:03:16 +0100 Subject: [PATCH 50/50] multi-attach: remote-path I/O errors are the takeover's, not per-node unavailable 2026-11-21 FTT2 bed (76b9e7e7), hard reset of sn3: 1.7 s before its LUNs were taken over, the event collector processed I/O-error events of nodes reaching those LUNs through sn3, and sent them a per-node "unavailable" (peer branch) and dropped their remote device. Nothing marked the devices available again after the takeover; on 1+0 every read through such a map fails. sn4's LVS failover then failed to read the superblock (ParityNoneRead init_subtask -3), its SPDK aborted (core dumped), the two LUNs it had just taken over moved again and the cluster suspended. - main_distr_event_collector: an I/O-error event of a node that reaches a multi-attach LUN through another node's export (served_elsewhere) is left to the LUN watchdog/takeover: no per-node unavailable, no remote-device removal. The serving node's own errors and regular clusters unchanged. - lun_takeover._switch_paths: once a node has its path to the new export, it is sent the device's ONLINE status (undoes anything sent in the gap). Co-Authored-By: Claude Opus 5.5 --- simplyblock_core/controllers/lun_takeover.py | 10 ++++++ .../services/main_distr_event_collector.py | 16 +++++++++ tests/unit/test_lun_takeover.py | 12 +++++++ tests/unit/test_multi_attach_device_events.py | 36 +++++++++++++++---- 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/simplyblock_core/controllers/lun_takeover.py b/simplyblock_core/controllers/lun_takeover.py index a95be1303e..73df44fb4a 100644 --- a/simplyblock_core/controllers/lun_takeover.py +++ b/simplyblock_core/controllers/lun_takeover.py @@ -932,6 +932,16 @@ def _one(node): # Usually: the path is already gone (the hold destructed it). logger.info("LUN takeover: old path %s of %s on %s not detached: %s", ip, name, node.get_id(), e) + # The node's distribs may have been told the device is unavailable + # while its old path failed; it is served again. On 1+0 a device left + # unavailable in one node's map fails every read there (2026-11-21 + # FTT2 bed: an LVS failover read on such a node aborted its SPDK). + try: + from simplyblock_core import distr_controller + distr_controller.send_dev_status_event(dev, NVMeDevice.STATUS_ONLINE, node) + except Exception as e: # noqa: BLE001 + logger.warning("LUN takeover: device status of %s not sent to %s: %s", + dev.get_id(), node.get_id(), e) return True if not nodes: diff --git a/simplyblock_core/services/main_distr_event_collector.py b/simplyblock_core/services/main_distr_event_collector.py index d0c7d00030..947722dc58 100644 --- a/simplyblock_core/services/main_distr_event_collector.py +++ b/simplyblock_core/services/main_distr_event_collector.py @@ -275,6 +275,22 @@ def process_device_event(event, logger): event.status = f'late_by_{int(time_delta.total_seconds())}s_skipping' return + if multi_attach.served_elsewhere(device_obj, event_node_obj.get_id()): + # A multi-attach LUN this node reaches through another node's + # export: its I/O errors are about that path. The LUN watchdog + # and the takeover handle a lost serving node and switch every + # node's path; marking the device unavailable for this node (or + # dropping its remote device) only turns a short hold into + # failed I/O on 1+0, and nothing marks it available again + # (2026-11-21 FTT2 bed: such events, processed during a + # takeover, left a device unavailable in a node's map; an LVS + # failover read there failed and its SPDK aborted). + logger.info(f"Multi-attach device {device_obj.get_id()} is served by " + f"{device_obj.serving_node()}, not by reporting node {event_node_obj.get_id()}: " + f"{event.message} left to the LUN takeover") + event.status = 'skipped:multi_attach_remote_path' + return + if device_obj.is_connection_in_progress_to_node(event_node_obj.get_id()): logger.warning("Connection attempt was found from node to device, sleeping 5 seconds") time.sleep(5) diff --git a/tests/unit/test_lun_takeover.py b/tests/unit/test_lun_takeover.py index ded9f65a1f..622af8ff35 100644 --- a/tests/unit/test_lun_takeover.py +++ b/tests/unit/test_lun_takeover.py @@ -379,6 +379,18 @@ def test_rerun_is_idempotent(self): self.assertEqual(self.dev(db).serving_node(), "node-b") self.assertTrue(b.spdk.ops("attach")) + def test_path_switch_marks_the_device_online_on_every_node(self): + # A node told "unavailable" during the gap must not keep it (1+0: + # every read there would fail). + from simplyblock_core import distr_controller + db, _ = _bed() + with patch.object(distr_controller, "send_dev_status_event") as sent: + self.assertEqual(self.run_takeover(db), lun_takeover.TAKEN_OVER) + targets = sorted(c.args[2].get_id() for c in sent.call_args_list + if c.args[1] == NVMeDevice.STATUS_ONLINE) + self.assertEqual(targets, sorted(n for n, x in db.nodes.items() + if n != "node-a" and x.status == StorageNode.STATUS_ONLINE)) + def test_rerun_after_partial_export_reuses_it(self): db, _ = _bed() standbys = [db.nodes[n] for n in ("node-b", "node-c", "node-d")] diff --git a/tests/unit/test_multi_attach_device_events.py b/tests/unit/test_multi_attach_device_events.py index a897e4fde6..5e88392e34 100644 --- a/tests/unit/test_multi_attach_device_events.py +++ b/tests/unit/test_multi_attach_device_events.py @@ -13,7 +13,7 @@ from simplyblock_core.services import main_distr_event_collector as collector -def _run(serving_node_id, multi=True): +def _run(serving_node_id, multi=True, event_node="node-1"): dev = NVMeDevice() dev.uuid = "dev-1" dev.node_id = "node-1" @@ -28,33 +28,55 @@ def _run(serving_node_id, multi=True): home.cluster_id = "c1" home.nvme_devices = [dev] + peer = MagicMock() + peer.get_id.return_value = "node-3" + peer.status = StorageNode.STATUS_ONLINE + peer.cluster_id = "c1" + peer.nvme_devices = [] + event = MagicMock() event.message = "error_read" - event.node_id = "node-1" + event.node_id = event_node event.storage_id = 3 event.object_dict = {} db = MagicMock() - db.get_storage_node_by_id.return_value = home - db.get_storage_nodes_by_cluster_id.return_value = [home] + db.get_storage_node_by_id.side_effect = lambda nid: peer if nid == "node-3" else home + db.get_storage_nodes_by_cluster_id.return_value = [home, peer] with patch.object(collector, "db", db), \ patch.object(collector.device_controller, "device_set_unavailable") as unavailable, \ patch.object(collector.device_controller, "device_set_io_error"), \ patch.object(collector.distr_controller, "send_dev_status_event") as peer_event, \ - patch.object(collector, "remove_remote_device_from_node"), \ + patch.object(collector, "remove_remote_device_from_node") as removed, \ patch.object(NVMeDevice, "lock_device_connection"), \ patch.object(NVMeDevice, "release_device_connection"), \ patch.object(NVMeDevice, "is_connection_in_progress_to_node", return_value=False): collector.process_device_event(event, collector.logger) + _run.removed = removed + _run.status = event.status return unavailable, peer_event class TestHomeNodeOfLunServedElsewhere(unittest.TestCase): - def test_error_is_handled_as_a_peer_path_error(self): + def test_home_node_error_is_left_to_the_takeover(self): unavailable, peer_event = _run(serving_node_id="node-2") unavailable.assert_not_called() - peer_event.assert_called_once() + peer_event.assert_not_called() + _run.removed.assert_not_called() + self.assertEqual(_run.status, "skipped:multi_attach_remote_path") + + def test_peer_error_is_left_to_the_takeover(self): + # 2026-11-21 FTT2 bed: per-node "unavailable" sent during a takeover + # stayed in the node's map; on 1+0 its reads failed and SPDK aborted. + unavailable, peer_event = _run(serving_node_id="node-2", event_node="node-3") + unavailable.assert_not_called() + peer_event.assert_not_called() + _run.removed.assert_not_called() + + def test_regular_cluster_peer_error_is_processed_as_before(self): + _run(serving_node_id="", multi=False, event_node="node-3") + self.assertNotEqual(_run.status, "skipped:multi_attach_remote_path") def test_home_node_serving_its_lun_fails_it(self): unavailable, peer_event = _run(serving_node_id="")